[MM-20979] Add first implementation of the Bleve search engine (#14562)

* [MM-20979] Add first implementation of the Bleve search engine

* Fix i18n

* Migrate searchengine utils tests

* Fix linter

* Don't add allTermsQ if both termQueries and notTermQueries are empty

* Fix test that should work if user is system admin

* Modify naming according to review comments

* Abstract getIndexDir function

* Extracting bleve engine name as a constant

* Merge both Indexer interfaces into one

* Add worker stopped message

* Allow worker to be started/stopped with config change

* Use constants for index names

* Modify test order

* Fix linter

* Trying to unlock the CI

vendor/github.com/RoaringBitmap/roaring/.drone.yml

@@ -0,0 +1,20 @@
+kind: pipeline
+name: default
+
+workspace:
+  base: /go
+  path: src/github.com/RoaringBitmap/roaring
+
+steps:
+- name: test
+  image: golang
+  commands:
+  - go get -t 
+  - go test 
+  - go test -race -run TestConcurrent*
+  - go build -tags appengine 
+  - go test -tags appengine 
+  - GOARCH=386 go build
+  - GOARCH=386 go test
+  - GOARCH=arm go build
+  - GOARCH=arm64 go build

vendor/github.com/RoaringBitmap/roaring/.gitignore

@@ -0,0 +1,6 @@
+*~
+roaring-fuzz.zip
+workdir
+coverage.out
+testdata/all3.classic
+testdata/all3.msgp.snappy

vendor/github.com/RoaringBitmap/roaring/.travis.yml

@@ -0,0 +1,37 @@
+language: go
+sudo: false
+install:
+- go get -t github.com/RoaringBitmap/roaring
+- go get -t golang.org/x/tools/cmd/cover
+- go get -t github.com/mattn/goveralls
+- go get -t github.com/mschoch/smat
+notifications:
+  email: false
+go:
+- "1.7.x"
+- "1.8.x"
+- "1.9.x"
+- "1.10.x"
+- "1.11.x"
+- "1.12.x"
+- "1.13.x"
+- tip
+
+# whitelist
+branches:
+  only:
+    - master
+script:
+- goveralls -v -service travis-ci -ignore arraycontainer_gen.go,bitmapcontainer_gen.go,rle16_gen.go,rle_gen.go,roaringarray_gen.go,rle.go || go test
+- go test -race -run TestConcurrent*
+- go build -tags appengine
+- go test -tags appengine
+- GOARCH=arm64 go build
+- GOARCH=386 go build
+- GOARCH=386 go test
+- GOARCH=arm go build
+- GOARCH=arm64 go build
+
+matrix:
+  allow_failures:
+  - go: tip

vendor/github.com/RoaringBitmap/roaring/AUTHORS

@@ -0,0 +1,11 @@
+# This is the official list of roaring authors for copyright purposes. 
+
+Todd Gruben (@tgruben),
+Daniel Lemire (@lemire),
+Elliot Murphy (@statik),
+Bob Potter (@bpot),
+Tyson Maly (@tvmaly),
+Will Glynn (@willglynn),
+Brent Pedersen (@brentp)
+Maciej Biłas (@maciej),
+Joe Nall (@joenall)

vendor/github.com/RoaringBitmap/roaring/CONTRIBUTORS

@@ -0,0 +1,16 @@
+# This is the official list of roaring contributors 
+
+Todd Gruben (@tgruben),
+Daniel Lemire (@lemire),
+Elliot Murphy (@statik),
+Bob Potter (@bpot),
+Tyson Maly (@tvmaly),
+Will Glynn (@willglynn),
+Brent Pedersen (@brentp),
+Jason E. Aten (@glycerine),
+Vali Malinoiu (@0x4139),
+Forud Ghafouri (@fzerorubigd),
+Joe Nall (@joenall),
+(@fredim),
+Edd Robinson (@e-dard),
+Alexander Petrov (@alldroll)

vendor/github.com/RoaringBitmap/roaring/LICENSE

@@ -0,0 +1,235 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2016 by the authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+================================================================================
+
+Portions of runcontainer.go are from the Go standard library, which is licensed
+under:
+
+Copyright (c) 2009 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+     copyright notice, this list of conditions and the following disclaimer
+     in the documentation and/or other materials provided with the
+     distribution.
+   * Neither the name of Google Inc. nor the names of its
+     contributors may be used to endorse or promote products derived from
+     this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/github.com/RoaringBitmap/roaring/LICENSE-2.0.txt

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2016 by the authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

vendor/github.com/RoaringBitmap/roaring/Makefile

@@ -0,0 +1,111 @@
+.PHONY: help all test format fmtcheck vet lint     qa deps clean nuke ser fetch-real-roaring-datasets
+
+
+
+
+
+
+
+
+# Display general help about this command
+help:
+	@echo ""
+	@echo "The following commands are available:"
+	@echo ""
+	@echo "    make qa          : Run all the tests"
+	@echo "    make test        : Run the unit tests"
+	@echo ""
+	@echo "    make format      : Format the source code"
+	@echo "    make fmtcheck    : Check if the source code has been formatted"
+	@echo "    make vet         : Check for suspicious constructs"
+	@echo "    make lint        : Check for style errors"
+	@echo ""
+	@echo "    make deps        : Get the dependencies"
+	@echo "    make clean       : Remove any build artifact"
+	@echo "    make nuke        : Deletes any intermediate file"
+	@echo ""
+	@echo "    make fuzz-smat   : Fuzzy testing with smat"
+	@echo "    make fuzz-stream : Fuzzy testing with stream deserialization"
+	@echo "    make fuzz-buffer : Fuzzy testing with buffer deserialization"
+	@echo ""
+
+# Alias for help target
+all: help
+test:
+	go test
+	go test -race -run TestConcurrent*
+# Format the source code
+format:
+	@find ./ -type f -name "*.go" -exec gofmt -w {} \;
+
+# Check if the source code has been formatted
+fmtcheck:
+	@mkdir -p target
+	@find ./ -type f -name "*.go" -exec gofmt -d {} \; | tee target/format.diff
+	@test ! -s target/format.diff || { echo "ERROR: the source code has not been formatted - please use 'make format' or 'gofmt'"; exit 1; }
+
+# Check for syntax errors
+vet:
+	GOPATH=$(GOPATH) go vet ./...
+
+# Check for style errors
+lint:
+	GOPATH=$(GOPATH) PATH=$(GOPATH)/bin:$(PATH) golint ./...
+
+
+
+
+
+# Alias to run all quality-assurance checks
+qa: fmtcheck test vet lint
+
+# --- INSTALL ---
+
+# Get the dependencies
+deps:
+	GOPATH=$(GOPATH) go get github.com/stretchr/testify
+	GOPATH=$(GOPATH) go get github.com/willf/bitset
+	GOPATH=$(GOPATH) go get github.com/golang/lint/golint
+	GOPATH=$(GOPATH) go get github.com/mschoch/smat
+	GOPATH=$(GOPATH) go get github.com/dvyukov/go-fuzz/go-fuzz
+	GOPATH=$(GOPATH) go get github.com/dvyukov/go-fuzz/go-fuzz-build
+	GOPATH=$(GOPATH) go get github.com/glycerine/go-unsnap-stream
+	GOPATH=$(GOPATH) go get github.com/philhofer/fwd
+	GOPATH=$(GOPATH) go get github.com/jtolds/gls
+
+fuzz-smat:
+	go test -tags=gofuzz -run=TestGenerateSmatCorpus
+	go-fuzz-build -func FuzzSmat github.com/RoaringBitmap/roaring
+	go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200
+
+
+fuzz-stream:
+	go-fuzz-build -func FuzzSerializationStream github.com/RoaringBitmap/roaring
+	go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200
+
+
+fuzz-buffer:
+	go-fuzz-build -func FuzzSerializationBuffer github.com/RoaringBitmap/roaring
+	go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200
+
+# Remove any build artifact
+clean:
+	GOPATH=$(GOPATH) go clean ./...
+
+# Deletes any intermediate file
+nuke:
+	rm -rf ./target
+	GOPATH=$(GOPATH) go clean -i ./...
+
+
+ser: 
+	go generate
+
+cover:
+	go test -coverprofile=coverage.out
+	go tool cover -html=coverage.out
+
+fetch-real-roaring-datasets:
+	# pull github.com/RoaringBitmap/real-roaring-datasets -> testdata/real-roaring-datasets
+	git submodule init
+	git submodule update

vendor/github.com/RoaringBitmap/roaring/README.md

@@ -0,0 +1,253 @@
+roaring [![Build Status](https://travis-ci.org/RoaringBitmap/roaring.png)](https://travis-ci.org/RoaringBitmap/roaring) [![Coverage Status](https://coveralls.io/repos/github/RoaringBitmap/roaring/badge.svg?branch=master)](https://coveralls.io/github/RoaringBitmap/roaring?branch=master) [![GoDoc](https://godoc.org/github.com/RoaringBitmap/roaring?status.svg)](https://godoc.org/github.com/RoaringBitmap/roaring) [![Go Report Card](https://goreportcard.com/badge/RoaringBitmap/roaring)](https://goreportcard.com/report/github.com/RoaringBitmap/roaring)
+[![Build Status](https://cloud.drone.io/api/badges/RoaringBitmap/roaring/status.svg)](https://cloud.drone.io/RoaringBitmap/roaring)
+=============
+
+This is a go version of the Roaring bitmap data structure. 
+
+
+
+Roaring bitmaps are used by several major systems such as [Apache Lucene][lucene] and derivative systems such as [Solr][solr] and
+[Elasticsearch][elasticsearch], [Apache Druid (Incubating)][druid], [LinkedIn Pinot][pinot], [Netflix Atlas][atlas],  [Apache Spark][spark], [OpenSearchServer][opensearchserver], [Cloud Torrent][cloudtorrent], [Whoosh][whoosh],  [Pilosa][pilosa],  [Microsoft Visual Studio Team Services (VSTS)][vsts], and eBay's [Apache Kylin][kylin].
+
+[lucene]: https://lucene.apache.org/
+[solr]: https://lucene.apache.org/solr/
+[elasticsearch]: https://www.elastic.co/products/elasticsearch
+[druid]: https://druid.apache.org/
+[spark]: https://spark.apache.org/
+[opensearchserver]: http://www.opensearchserver.com
+[cloudtorrent]: https://github.com/jpillora/cloud-torrent
+[whoosh]: https://bitbucket.org/mchaput/whoosh/wiki/Home
+[pilosa]: https://www.pilosa.com/
+[kylin]: http://kylin.apache.org/
+[pinot]: http://github.com/linkedin/pinot/wiki
+[vsts]: https://www.visualstudio.com/team-services/
+[atlas]: https://github.com/Netflix/atlas
+
+Roaring bitmaps are found to work well in many important applications:
+
+> Use Roaring for bitmap compression whenever possible. Do not use other bitmap compression methods ([Wang et al., SIGMOD 2017](http://db.ucsd.edu/wp-content/uploads/2017/03/sidm338-wangA.pdf))
+
+
+The ``roaring`` Go library is used by
+* [Cloud Torrent](https://github.com/jpillora/cloud-torrent)
+* [runv](https://github.com/hyperhq/runv)
+* [InfluxDB](https://www.influxdata.com)
+* [Pilosa](https://www.pilosa.com/)
+* [Bleve](http://www.blevesearch.com)
+* [lindb](https://github.com/lindb/lindb)
+* [Elasticell](https://github.com/deepfabric/elasticell)
+* [SourceGraph](https://github.com/sourcegraph/sourcegraph)
+* [M3](https://github.com/m3db/m3)
+* [trident](https://github.com/NetApp/trident)
+
+
+This library is used in production in several systems, it is part of the [Awesome Go collection](https://awesome-go.com).
+
+
+There are also  [Java](https://github.com/RoaringBitmap/RoaringBitmap) and [C/C++](https://github.com/RoaringBitmap/CRoaring) versions.  The Java, C, C++ and Go version are binary compatible: e.g,  you can save bitmaps
+from a Java program and load them back in Go, and vice versa. We have a [format specification](https://github.com/RoaringBitmap/RoaringFormatSpec).
+
+
+This code is licensed under Apache License, Version 2.0 (ASL2.0).
+
+Copyright 2016-... by the authors.
+
+
+### References
+
+- Daniel Lemire, Owen Kaser, Nathan Kurz, Luca Deri, Chris O'Hara, François Saint-Jacques, Gregory Ssi-Yan-Kai, Roaring Bitmaps: Implementation of an Optimized Software Library, Software: Practice and Experience 48 (4), 2018 [arXiv:1709.07821](https://arxiv.org/abs/1709.07821)
+-  Samy Chambi, Daniel Lemire, Owen Kaser, Robert Godin,
+Better bitmap performance with Roaring bitmaps,
+Software: Practice and Experience 46 (5), 2016.
+http://arxiv.org/abs/1402.6407 This paper used data from http://lemire.me/data/realroaring2014.html
+- Daniel Lemire, Gregory Ssi-Yan-Kai, Owen Kaser, Consistently faster and smaller compressed bitmaps with Roaring, Software: Practice and Experience 46 (11), 2016. http://arxiv.org/abs/1603.06549
+
+
+### Dependencies
+
+Dependencies are fetched automatically by giving the `-t` flag to `go get`.
+
+they include
+  - github.com/willf/bitset
+  - github.com/mschoch/smat
+  - github.com/glycerine/go-unsnap-stream
+  - github.com/philhofer/fwd
+  - github.com/jtolds/gls
+
+Note that the smat library requires Go 1.6 or better.
+
+#### Installation
+
+  - go get -t github.com/RoaringBitmap/roaring
+
+
+### Example
+
+Here is a simplified but complete example:
+
+```go
+package main
+
+import (
+    "fmt"
+    "github.com/RoaringBitmap/roaring"
+    "bytes"
+)
+
+
+func main() {
+    // example inspired by https://github.com/fzandona/goroar
+    fmt.Println("==roaring==")
+    rb1 := roaring.BitmapOf(1, 2, 3, 4, 5, 100, 1000)
+    fmt.Println(rb1.String())
+
+    rb2 := roaring.BitmapOf(3, 4, 1000)
+    fmt.Println(rb2.String())
+
+    rb3 := roaring.New()
+    fmt.Println(rb3.String())
+
+    fmt.Println("Cardinality: ", rb1.GetCardinality())
+
+    fmt.Println("Contains 3? ", rb1.Contains(3))
+
+    rb1.And(rb2)
+
+    rb3.Add(1)
+    rb3.Add(5)
+
+    rb3.Or(rb1)
+
+    // computes union of the three bitmaps in parallel using 4 workers  
+    roaring.ParOr(4, rb1, rb2, rb3)
+    // computes intersection of the three bitmaps in parallel using 4 workers  
+    roaring.ParAnd(4, rb1, rb2, rb3)
+
+
+    // prints 1, 3, 4, 5, 1000
+    i := rb3.Iterator()
+    for i.HasNext() {
+        fmt.Println(i.Next())
+    }
+    fmt.Println()
+
+    // next we include an example of serialization
+    buf := new(bytes.Buffer)
+    rb1.WriteTo(buf) // we omit error handling
+    newrb:= roaring.New()
+    newrb.ReadFrom(buf)
+    if rb1.Equals(newrb) {
+    	fmt.Println("I wrote the content to a byte stream and read it back.")
+    }
+    // you can iterate over bitmaps using ReverseIterator(), Iterator, ManyIterator()
+}
+```
+
+If you wish to use serialization and handle errors, you might want to
+consider the following sample of code:
+
+```go
+	rb := BitmapOf(1, 2, 3, 4, 5, 100, 1000)
+	buf := new(bytes.Buffer)
+	size,err:=rb.WriteTo(buf)
+	if err != nil {
+		t.Errorf("Failed writing")
+	}
+	newrb:= New()
+	size,err=newrb.ReadFrom(buf)
+	if err != nil {
+		t.Errorf("Failed reading")
+	}
+	if ! rb.Equals(newrb) {
+		t.Errorf("Cannot retrieve serialized version")
+	}
+```
+
+Given N integers in [0,x), then the serialized size in bytes of
+a Roaring bitmap should never exceed this bound:
+
+`` 8 + 9 * ((long)x+65535)/65536 + 2 * N ``
+
+That is, given a fixed overhead for the universe size (x), Roaring
+bitmaps never use more than 2 bytes per integer. You can call
+``BoundSerializedSizeInBytes`` for a more precise estimate.
+
+
+### Documentation
+
+Current documentation is available at http://godoc.org/github.com/RoaringBitmap/roaring
+
+### Goroutine safety
+
+In general, it should not generally be considered safe to access
+the same bitmaps using different goroutines--they are left
+unsynchronized for performance. Should you want to access
+a Bitmap from more than one goroutine, you should
+provide synchronization. Typically this is done by using channels to pass
+the *Bitmap around (in Go style; so there is only ever one owner),
+or by using `sync.Mutex` to serialize operations on Bitmaps.
+
+### Coverage
+
+We test our software. For a report on our test coverage, see
+
+https://coveralls.io/github/RoaringBitmap/roaring?branch=master
+
+### Benchmark
+
+Type
+
+         go test -bench Benchmark -run -
+         
+To run benchmarks on [Real Roaring Datasets](https://github.com/RoaringBitmap/real-roaring-datasets)
+run the following:
+
+```sh
+go get github.com/RoaringBitmap/real-roaring-datasets
+BENCH_REAL_DATA=1 go test -bench BenchmarkRealData -run -
+```
+
+### Iterative use
+
+You can use roaring with gore:
+
+- go get -u github.com/motemen/gore
+- Make sure that ``$GOPATH/bin`` is in your ``$PATH``.
+- go get github.com/RoaringBitmap/roaring
+
+```go
+$ gore
+gore version 0.2.6  :help for help
+gore> :import github.com/RoaringBitmap/roaring
+gore> x:=roaring.New()
+gore> x.Add(1)
+gore> x.String()
+"{1}"
+```
+
+
+### Fuzzy testing
+
+You can help us test further the library with fuzzy testing:
+
+         go get github.com/dvyukov/go-fuzz/go-fuzz
+         go get github.com/dvyukov/go-fuzz/go-fuzz-build
+         go test -tags=gofuzz -run=TestGenerateSmatCorpus
+         go-fuzz-build github.com/RoaringBitmap/roaring
+         go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200
+
+Let it run, and if the # of crashers is > 0, check out the reports in
+the workdir where you should be able to find the panic goroutine stack
+traces.
+
+### Alternative in Go
+
+There is a Go version wrapping the C/C++ implementation https://github.com/RoaringBitmap/gocroaring
+
+For an alternative implementation in Go, see https://github.com/fzandona/goroar
+The two versions were written independently.
+
+
+### Mailing list/discussion group
+
+https://groups.google.com/forum/#!forum/roaring-bitmaps

vendor/github.com/RoaringBitmap/roaring/arraycontainer.go

@@ -0,0 +1,980 @@
+package roaring
+
+import (
+	"fmt"
+)
+
+//go:generate msgp -unexported
+
+type arrayContainer struct {
+	content []uint16
+}
+
+func (ac *arrayContainer) String() string {
+	s := "{"
+	for it := ac.getShortIterator(); it.hasNext(); {
+		s += fmt.Sprintf("%v, ", it.next())
+	}
+	return s + "}"
+}
+
+func (ac *arrayContainer) fillLeastSignificant16bits(x []uint32, i int, mask uint32) {
+	for k := 0; k < len(ac.content); k++ {
+		x[k+i] = uint32(ac.content[k]) | mask
+	}
+}
+
+func (ac *arrayContainer) iterate(cb func(x uint16) bool) bool {
+	iterator := shortIterator{ac.content, 0}
+
+	for iterator.hasNext() {
+		if !cb(iterator.next()) {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (ac *arrayContainer) getShortIterator() shortPeekable {
+	return &shortIterator{ac.content, 0}
+}
+
+func (ac *arrayContainer) getReverseIterator() shortIterable {
+	return &reverseIterator{ac.content, len(ac.content) - 1}
+}
+
+func (ac *arrayContainer) getManyIterator() manyIterable {
+	return &shortIterator{ac.content, 0}
+}
+
+func (ac *arrayContainer) minimum() uint16 {
+	return ac.content[0] // assume not empty
+}
+
+func (ac *arrayContainer) maximum() uint16 {
+	return ac.content[len(ac.content)-1] // assume not empty
+}
+
+func (ac *arrayContainer) getSizeInBytes() int {
+	return ac.getCardinality() * 2
+}
+
+func (ac *arrayContainer) serializedSizeInBytes() int {
+	return ac.getCardinality() * 2
+}
+
+func arrayContainerSizeInBytes(card int) int {
+	return card * 2
+}
+
+// add the values in the range [firstOfRange,endx)
+func (ac *arrayContainer) iaddRange(firstOfRange, endx int) container {
+	if firstOfRange >= endx {
+		return ac
+	}
+	indexstart := binarySearch(ac.content, uint16(firstOfRange))
+	if indexstart < 0 {
+		indexstart = -indexstart - 1
+	}
+	indexend := binarySearch(ac.content, uint16(endx-1))
+	if indexend < 0 {
+		indexend = -indexend - 1
+	} else {
+		indexend++
+	}
+	rangelength := endx - firstOfRange
+	newcardinality := indexstart + (ac.getCardinality() - indexend) + rangelength
+	if newcardinality > arrayDefaultMaxSize {
+		a := ac.toBitmapContainer()
+		return a.iaddRange(firstOfRange, endx)
+	}
+	if cap(ac.content) < newcardinality {
+		tmp := make([]uint16, newcardinality, newcardinality)
+		copy(tmp[:indexstart], ac.content[:indexstart])
+		copy(tmp[indexstart+rangelength:], ac.content[indexend:])
+
+		ac.content = tmp
+	} else {
+		ac.content = ac.content[:newcardinality]
+		copy(ac.content[indexstart+rangelength:], ac.content[indexend:])
+
+	}
+	for k := 0; k < rangelength; k++ {
+		ac.content[k+indexstart] = uint16(firstOfRange + k)
+	}
+	return ac
+}
+
+// remove the values in the range [firstOfRange,endx)
+func (ac *arrayContainer) iremoveRange(firstOfRange, endx int) container {
+	if firstOfRange >= endx {
+		return ac
+	}
+	indexstart := binarySearch(ac.content, uint16(firstOfRange))
+	if indexstart < 0 {
+		indexstart = -indexstart - 1
+	}
+	indexend := binarySearch(ac.content, uint16(endx-1))
+	if indexend < 0 {
+		indexend = -indexend - 1
+	} else {
+		indexend++
+	}
+	rangelength := indexend - indexstart
+	answer := ac
+	copy(answer.content[indexstart:], ac.content[indexstart+rangelength:])
+	answer.content = answer.content[:ac.getCardinality()-rangelength]
+	return answer
+}
+
+// flip the values in the range [firstOfRange,endx)
+func (ac *arrayContainer) not(firstOfRange, endx int) container {
+	if firstOfRange >= endx {
+		return ac.clone()
+	}
+	return ac.notClose(firstOfRange, endx-1) // remove everything in [firstOfRange,endx-1]
+}
+
+// flip the values in the range [firstOfRange,lastOfRange]
+func (ac *arrayContainer) notClose(firstOfRange, lastOfRange int) container {
+	if firstOfRange > lastOfRange { // unlike add and remove, not uses an inclusive range [firstOfRange,lastOfRange]
+		return ac.clone()
+	}
+
+	// determine the span of array indices to be affected^M
+	startIndex := binarySearch(ac.content, uint16(firstOfRange))
+	if startIndex < 0 {
+		startIndex = -startIndex - 1
+	}
+	lastIndex := binarySearch(ac.content, uint16(lastOfRange))
+	if lastIndex < 0 {
+		lastIndex = -lastIndex - 2
+	}
+	currentValuesInRange := lastIndex - startIndex + 1
+	spanToBeFlipped := lastOfRange - firstOfRange + 1
+	newValuesInRange := spanToBeFlipped - currentValuesInRange
+	cardinalityChange := newValuesInRange - currentValuesInRange
+	newCardinality := len(ac.content) + cardinalityChange
+	if newCardinality > arrayDefaultMaxSize {
+		return ac.toBitmapContainer().not(firstOfRange, lastOfRange+1)
+	}
+	answer := newArrayContainer()
+	answer.content = make([]uint16, newCardinality, newCardinality) //a hack for sure
+
+	copy(answer.content, ac.content[:startIndex])
+	outPos := startIndex
+	inPos := startIndex
+	valInRange := firstOfRange
+	for ; valInRange <= lastOfRange && inPos <= lastIndex; valInRange++ {
+		if uint16(valInRange) != ac.content[inPos] {
+			answer.content[outPos] = uint16(valInRange)
+			outPos++
+		} else {
+			inPos++
+		}
+	}
+
+	for ; valInRange <= lastOfRange; valInRange++ {
+		answer.content[outPos] = uint16(valInRange)
+		outPos++
+	}
+
+	for i := lastIndex + 1; i < len(ac.content); i++ {
+		answer.content[outPos] = ac.content[i]
+		outPos++
+	}
+	answer.content = answer.content[:newCardinality]
+	return answer
+
+}
+
+func (ac *arrayContainer) equals(o container) bool {
+
+	srb, ok := o.(*arrayContainer)
+	if ok {
+		// Check if the containers are the same object.
+		if ac == srb {
+			return true
+		}
+
+		if len(srb.content) != len(ac.content) {
+			return false
+		}
+
+		for i, v := range ac.content {
+			if v != srb.content[i] {
+				return false
+			}
+		}
+		return true
+	}
+
+	// use generic comparison
+	bCard := o.getCardinality()
+	aCard := ac.getCardinality()
+	if bCard != aCard {
+		return false
+	}
+
+	ait := ac.getShortIterator()
+	bit := o.getShortIterator()
+	for ait.hasNext() {
+		if bit.next() != ait.next() {
+			return false
+		}
+	}
+	return true
+}
+
+func (ac *arrayContainer) toBitmapContainer() *bitmapContainer {
+	bc := newBitmapContainer()
+	bc.loadData(ac)
+	return bc
+
+}
+func (ac *arrayContainer) iadd(x uint16) (wasNew bool) {
+	// Special case adding to the end of the container.
+	l := len(ac.content)
+	if l > 0 && l < arrayDefaultMaxSize && ac.content[l-1] < x {
+		ac.content = append(ac.content, x)
+		return true
+	}
+
+	loc := binarySearch(ac.content, x)
+
+	if loc < 0 {
+		s := ac.content
+		i := -loc - 1
+		s = append(s, 0)
+		copy(s[i+1:], s[i:])
+		s[i] = x
+		ac.content = s
+		return true
+	}
+	return false
+}
+
+func (ac *arrayContainer) iaddReturnMinimized(x uint16) container {
+	// Special case adding to the end of the container.
+	l := len(ac.content)
+	if l > 0 && l < arrayDefaultMaxSize && ac.content[l-1] < x {
+		ac.content = append(ac.content, x)
+		return ac
+	}
+
+	loc := binarySearch(ac.content, x)
+
+	if loc < 0 {
+		if len(ac.content) >= arrayDefaultMaxSize {
+			a := ac.toBitmapContainer()
+			a.iadd(x)
+			return a
+		}
+		s := ac.content
+		i := -loc - 1
+		s = append(s, 0)
+		copy(s[i+1:], s[i:])
+		s[i] = x
+		ac.content = s
+	}
+	return ac
+}
+
+// iremoveReturnMinimized is allowed to change the return type to minimize storage.
+func (ac *arrayContainer) iremoveReturnMinimized(x uint16) container {
+	ac.iremove(x)
+	return ac
+}
+
+func (ac *arrayContainer) iremove(x uint16) bool {
+	loc := binarySearch(ac.content, x)
+	if loc >= 0 {
+		s := ac.content
+		s = append(s[:loc], s[loc+1:]...)
+		ac.content = s
+		return true
+	}
+	return false
+}
+
+func (ac *arrayContainer) remove(x uint16) container {
+	out := &arrayContainer{make([]uint16, len(ac.content))}
+	copy(out.content, ac.content[:])
+
+	loc := binarySearch(out.content, x)
+	if loc >= 0 {
+		s := out.content
+		s = append(s[:loc], s[loc+1:]...)
+		out.content = s
+	}
+	return out
+}
+
+func (ac *arrayContainer) or(a container) container {
+	switch x := a.(type) {
+	case *arrayContainer:
+		return ac.orArray(x)
+	case *bitmapContainer:
+		return x.orArray(ac)
+	case *runContainer16:
+		if x.isFull() {
+			return x.clone()
+		}
+		return x.orArray(ac)
+	}
+	panic("unsupported container type")
+}
+
+func (ac *arrayContainer) orCardinality(a container) int {
+	switch x := a.(type) {
+	case *arrayContainer:
+		return ac.orArrayCardinality(x)
+	case *bitmapContainer:
+		return x.orArrayCardinality(ac)
+	case *runContainer16:
+		return x.orArrayCardinality(ac)
+	}
+	panic("unsupported container type")
+}
+
+func (ac *arrayContainer) ior(a container) container {
+	switch x := a.(type) {
+	case *arrayContainer:
+		return ac.iorArray(x)
+	case *bitmapContainer:
+		return a.(*bitmapContainer).orArray(ac)
+		//return ac.iorBitmap(x) // note: this does not make sense
+	case *runContainer16:
+		if x.isFull() {
+			return x.clone()
+		}
+		return ac.iorRun16(x)
+	}
+	panic("unsupported container type")
+}
+
+func (ac *arrayContainer) iorArray(value2 *arrayContainer) container {
+	value1 := ac
+	len1 := value1.getCardinality()
+	len2 := value2.getCardinality()
+	maxPossibleCardinality := len1 + len2
+	if maxPossibleCardinality > arrayDefaultMaxSize { // it could be a bitmap!
+		bc := newBitmapContainer()
+		for k := 0; k < len(value2.content); k++ {
+			v := value2.content[k]
+			i := uint(v) >> 6
+			mask := uint64(1) << (v % 64)
+			bc.bitmap[i] |= mask
+		}
+		for k := 0; k < len(ac.content); k++ {
+			v := ac.content[k]
+			i := uint(v) >> 6
+			mask := uint64(1) << (v % 64)
+			bc.bitmap[i] |= mask
+		}
+		bc.cardinality = int(popcntSlice(bc.bitmap))
+		if bc.cardinality <= arrayDefaultMaxSize {
+			return bc.toArrayContainer()
+		}
+		return bc
+	}
+	if maxPossibleCardinality > cap(value1.content) {
+		newcontent := make([]uint16, 0, maxPossibleCardinality)
+		copy(newcontent[len2:maxPossibleCardinality], ac.content[0:len1])
+		ac.content = newcontent
+	} else {
+		copy(ac.content[len2:maxPossibleCardinality], ac.content[0:len1])
+	}
+	nl := union2by2(value1.content[len2:maxPossibleCardinality], value2.content, ac.content)
+	ac.content = ac.content[:nl] // reslice to match actual used capacity
+	return ac
+}
+
+// Note: such code does not make practical sense, except for lazy evaluations
+func (ac *arrayContainer) iorBitmap(bc2 *bitmapContainer) container {
+	bc1 := ac.toBitmapContainer()
+	bc1.iorBitmap(bc2)
+	*ac = *newArrayContainerFromBitmap(bc1)
+	return ac
+}
+
+func (ac *arrayContainer) iorRun16(rc *runContainer16) container {
+	bc1 := ac.toBitmapContainer()
+	bc2 := rc.toBitmapContainer()
+	bc1.iorBitmap(bc2)
+	*ac = *newArrayContainerFromBitmap(bc1)
+	return ac
+}
+
+func (ac *arrayContainer) lazyIOR(a container) container {
+	switch x := a.(type) {
+	case *arrayContainer:
+		return ac.lazyIorArray(x)
+	case *bitmapContainer:
+		return ac.lazyIorBitmap(x)
+	case *runContainer16:
+		if x.isFull() {
+			return x.clone()
+		}
+		return ac.lazyIorRun16(x)
+
+	}
+	panic("unsupported container type")
+}
+
+func (ac *arrayContainer) lazyIorArray(ac2 *arrayContainer) container {
+	// TODO actually make this lazy
+	return ac.iorArray(ac2)
+}
+
+func (ac *arrayContainer) lazyIorBitmap(bc *bitmapContainer) container {
+	// TODO actually make this lazy
+	return ac.iorBitmap(bc)
+}
+
+func (ac *arrayContainer) lazyIorRun16(rc *runContainer16) container {
+	// TODO actually make this lazy
+	return ac.iorRun16(rc)
+}
+
+func (ac *arrayContainer) lazyOR(a container) container {
+	switch x := a.(type) {
+	case *arrayContainer:
+		return ac.lazyorArray(x)
+	case *bitmapContainer:
+		return a.lazyOR(ac)
+	case *runContainer16:
+		if x.isFull() {
+			return x.clone()
+		}
+		return x.orArray(ac)
+	}
+	panic("unsupported container type")
+}
+
+func (ac *arrayContainer) orArray(value2 *arrayContainer) container {
+	value1 := ac
+	maxPossibleCardinality := value1.getCardinality() + value2.getCardinality()
+	if maxPossibleCardinality > arrayDefaultMaxSize { // it could be a bitmap!
+		bc := newBitmapContainer()
+		for k := 0; k < len(value2.content); k++ {
+			v := value2.content[k]
+			i := uint(v) >> 6
+			mask := uint64(1) << (v % 64)
+			bc.bitmap[i] |= mask
+		}
+		for k := 0; k < len(ac.content); k++ {
+			v := ac.content[k]
+			i := uint(v) >> 6
+			mask := uint64(1) << (v % 64)
+			bc.bitmap[i] |= mask
+		}
+		bc.cardinality = int(popcntSlice(bc.bitmap))
+		if bc.cardinality <= arrayDefaultMaxSize {
+			return bc.toArrayContainer()
+		}
+		return bc
+	}
+	answer := newArrayContainerCapacity(maxPossibleCardinality)
+	nl := union2by2(value1.content, value2.content, answer.content)
+	answer.content = answer.content[:nl] // reslice to match actual used capacity
+	return answer
+}
+
+func (ac *arrayContainer) orArrayCardinality(value2 *arrayContainer) int {
+	return union2by2Cardinality(ac.content, value2.content)
+}
+
+func (ac *arrayContainer) lazyorArray(value2 *arrayContainer) container {
+	value1 := ac
+	maxPossibleCardinality := value1.getCardinality() + value2.getCardinality()
+	if maxPossibleCardinality > arrayLazyLowerBound { // it could be a bitmap!^M
+		bc := newBitmapContainer()
+		for k := 0; k < len(value2.content); k++ {
+			v := value2.content[k]
+			i := uint(v) >> 6
+			mask := uint64(1) << (v % 64)
+			bc.bitmap[i] |= mask
+		}
+		for k := 0; k < len(ac.content); k++ {
+			v := ac.content[k]
+			i := uint(v) >> 6
+			mask := uint64(1) << (v % 64)
+			bc.bitmap[i] |= mask
+		}
+		bc.cardinality = invalidCardinality
+		return bc
+	}
+	answer := newArrayContainerCapacity(maxPossibleCardinality)
+	nl := union2by2(value1.content, value2.content, answer.content)
+	answer.content = answer.content[:nl] // reslice to match actual used capacity
+	return answer
+}
+
+func (ac *arrayContainer) and(a container) container {
+	switch x := a.(type) {
+	case *arrayContainer:
+		return ac.andArray(x)
+	case *bitmapContainer:
+		return x.and(ac)
+	case *runContainer16:
+		if x.isFull() {
+			return ac.clone()
+		}
+		return x.andArray(ac)
+	}
+	panic("unsupported container type")
+}
+
+func (ac *arrayContainer) andCardinality(a container) int {
+	switch x := a.(type) {
+	case *arrayContainer:
+		return ac.andArrayCardinality(x)
+	case *bitmapContainer:
+		return x.andCardinality(ac)
+	case *runContainer16:
+		return x.andArrayCardinality(ac)
+	}
+	panic("unsupported container type")
+}
+
+func (ac *arrayContainer) intersects(a container) bool {
+	switch x := a.(type) {
+	case *arrayContainer:
+		return ac.intersectsArray(x)
+	case *bitmapContainer:
+		return x.intersects(ac)
+	case *runContainer16:
+		return x.intersects(ac)
+	}
+	panic("unsupported container type")
+}
+
+func (ac *arrayContainer) iand(a container) container {
+	switch x := a.(type) {
+	case *arrayContainer:
+		return ac.iandArray(x)
+	case *bitmapContainer:
+		return ac.iandBitmap(x)
+	case *runContainer16:
+		if x.isFull() {
+			return ac
+		}
+		return x.andArray(ac)
+	}
+	panic("unsupported container type")
+}
+
+func (ac *arrayContainer) iandBitmap(bc *bitmapContainer) container {
+	pos := 0
+	c := ac.getCardinality()
+	for k := 0; k < c; k++ {
+		// branchless
+		v := ac.content[k]
+		ac.content[pos] = v
+		pos += int(bc.bitValue(v))
+	}
+	ac.content = ac.content[:pos]
+	return ac
+
+}
+
+func (ac *arrayContainer) xor(a container) container {
+	switch x := a.(type) {
+	case *arrayContainer:
+		return ac.xorArray(x)
+	case *bitmapContainer:
+		return a.xor(ac)
+	case *runContainer16:
+		return x.xorArray(ac)
+	}
+	panic("unsupported container type")
+}
+
+func (ac *arrayContainer) xorArray(value2 *arrayContainer) container {
+	value1 := ac
+	totalCardinality := value1.getCardinality() + value2.getCardinality()
+	if totalCardinality > arrayDefaultMaxSize { // it could be a bitmap!
+		bc := newBitmapContainer()
+		for k := 0; k < len(value2.content); k++ {
+			v := value2.content[k]
+			i := uint(v) >> 6
+			bc.bitmap[i] ^= (uint64(1) << (v % 64))
+		}
+		for k := 0; k < len(ac.content); k++ {
+			v := ac.content[k]
+			i := uint(v) >> 6
+			bc.bitmap[i] ^= (uint64(1) << (v % 64))
+		}
+		bc.computeCardinality()
+		if bc.cardinality <= arrayDefaultMaxSize {
+			return bc.toArrayContainer()
+		}
+		return bc
+	}
+	desiredCapacity := totalCardinality
+	answer := newArrayContainerCapacity(desiredCapacity)
+	length := exclusiveUnion2by2(value1.content, value2.content, answer.content)
+	answer.content = answer.content[:length]
+	return answer
+
+}
+
+func (ac *arrayContainer) andNot(a container) container {
+	switch x := a.(type) {
+	case *arrayContainer:
+		return ac.andNotArray(x)
+	case *bitmapContainer:
+		return ac.andNotBitmap(x)
+	case *runContainer16:
+		return ac.andNotRun16(x)
+	}
+	panic("unsupported container type")
+}
+
+func (ac *arrayContainer) andNotRun16(rc *runContainer16) container {
+	acb := ac.toBitmapContainer()
+	rcb := rc.toBitmapContainer()
+	return acb.andNotBitmap(rcb)
+}
+
+func (ac *arrayContainer) iandNot(a container) container {
+	switch x := a.(type) {
+	case *arrayContainer:
+		return ac.iandNotArray(x)
+	case *bitmapContainer:
+		return ac.iandNotBitmap(x)
+	case *runContainer16:
+		return ac.iandNotRun16(x)
+	}
+	panic("unsupported container type")
+}
+
+func (ac *arrayContainer) iandNotRun16(rc *runContainer16) container {
+	rcb := rc.toBitmapContainer()
+	acb := ac.toBitmapContainer()
+	acb.iandNotBitmapSurely(rcb)
+	*ac = *(acb.toArrayContainer())
+	return ac
+}
+
+func (ac *arrayContainer) andNotArray(value2 *arrayContainer) container {
+	value1 := ac
+	desiredcapacity := value1.getCardinality()
+	answer := newArrayContainerCapacity(desiredcapacity)
+	length := difference(value1.content, value2.content, answer.content)
+	answer.content = answer.content[:length]
+	return answer
+}
+
+func (ac *arrayContainer) iandNotArray(value2 *arrayContainer) container {
+	length := difference(ac.content, value2.content, ac.content)
+	ac.content = ac.content[:length]
+	return ac
+}
+
+func (ac *arrayContainer) andNotBitmap(value2 *bitmapContainer) container {
+	desiredcapacity := ac.getCardinality()
+	answer := newArrayContainerCapacity(desiredcapacity)
+	answer.content = answer.content[:desiredcapacity]
+	pos := 0
+	for _, v := range ac.content {
+		answer.content[pos] = v
+		pos += 1 - int(value2.bitValue(v))
+	}
+	answer.content = answer.content[:pos]
+	return answer
+}
+
+func (ac *arrayContainer) andBitmap(value2 *bitmapContainer) container {
+	desiredcapacity := ac.getCardinality()
+	answer := newArrayContainerCapacity(desiredcapacity)
+	answer.content = answer.content[:desiredcapacity]
+	pos := 0
+	for _, v := range ac.content {
+		answer.content[pos] = v
+		pos += int(value2.bitValue(v))
+	}
+	answer.content = answer.content[:pos]
+	return answer
+}
+
+func (ac *arrayContainer) iandNotBitmap(value2 *bitmapContainer) container {
+	pos := 0
+	for _, v := range ac.content {
+		ac.content[pos] = v
+		pos += 1 - int(value2.bitValue(v))
+	}
+	ac.content = ac.content[:pos]
+	return ac
+}
+
+func copyOf(array []uint16, size int) []uint16 {
+	result := make([]uint16, size)
+	for i, x := range array {
+		if i == size {
+			break
+		}
+		result[i] = x
+	}
+	return result
+}
+
+// flip the values in the range [firstOfRange,endx)
+func (ac *arrayContainer) inot(firstOfRange, endx int) container {
+	if firstOfRange >= endx {
+		return ac
+	}
+	return ac.inotClose(firstOfRange, endx-1) // remove everything in [firstOfRange,endx-1]
+}
+
+// flip the values in the range [firstOfRange,lastOfRange]
+func (ac *arrayContainer) inotClose(firstOfRange, lastOfRange int) container {
+	if firstOfRange > lastOfRange { // unlike add and remove, not uses an inclusive range [firstOfRange,lastOfRange]
+		return ac
+	}
+	// determine the span of array indices to be affected
+	startIndex := binarySearch(ac.content, uint16(firstOfRange))
+	if startIndex < 0 {
+		startIndex = -startIndex - 1
+	}
+	lastIndex := binarySearch(ac.content, uint16(lastOfRange))
+	if lastIndex < 0 {
+		lastIndex = -lastIndex - 1 - 1
+	}
+	currentValuesInRange := lastIndex - startIndex + 1
+	spanToBeFlipped := lastOfRange - firstOfRange + 1
+
+	newValuesInRange := spanToBeFlipped - currentValuesInRange
+	buffer := make([]uint16, newValuesInRange)
+	cardinalityChange := newValuesInRange - currentValuesInRange
+	newCardinality := len(ac.content) + cardinalityChange
+	if cardinalityChange > 0 {
+		if newCardinality > len(ac.content) {
+			if newCardinality > arrayDefaultMaxSize {
+				bcRet := ac.toBitmapContainer()
+				bcRet.inot(firstOfRange, lastOfRange+1)
+				*ac = *bcRet.toArrayContainer()
+				return bcRet
+			}
+			ac.content = copyOf(ac.content, newCardinality)
+		}
+		base := lastIndex + 1
+		copy(ac.content[lastIndex+1+cardinalityChange:], ac.content[base:base+len(ac.content)-1-lastIndex])
+		ac.negateRange(buffer, startIndex, lastIndex, firstOfRange, lastOfRange+1)
+	} else { // no expansion needed
+		ac.negateRange(buffer, startIndex, lastIndex, firstOfRange, lastOfRange+1)
+		if cardinalityChange < 0 {
+
+			for i := startIndex + newValuesInRange; i < newCardinality; i++ {
+				ac.content[i] = ac.content[i-cardinalityChange]
+			}
+		}
+	}
+	ac.content = ac.content[:newCardinality]
+	return ac
+}
+
+func (ac *arrayContainer) negateRange(buffer []uint16, startIndex, lastIndex, startRange, lastRange int) {
+	// compute the negation into buffer
+	outPos := 0
+	inPos := startIndex // value here always >= valInRange,
+	// until it is exhausted
+	// n.b., we can start initially exhausted.
+
+	valInRange := startRange
+	for ; valInRange < lastRange && inPos <= lastIndex; valInRange++ {
+		if uint16(valInRange) != ac.content[inPos] {
+			buffer[outPos] = uint16(valInRange)
+			outPos++
+		} else {
+			inPos++
+		}
+	}
+
+	// if there are extra items (greater than the biggest
+	// pre-existing one in range), buffer them
+	for ; valInRange < lastRange; valInRange++ {
+		buffer[outPos] = uint16(valInRange)
+		outPos++
+	}
+
+	if outPos != len(buffer) {
+		panic("negateRange: internal bug")
+	}
+
+	for i, item := range buffer {
+		ac.content[i+startIndex] = item
+	}
+}
+
+func (ac *arrayContainer) isFull() bool {
+	return false
+}
+
+func (ac *arrayContainer) andArray(value2 *arrayContainer) container {
+	desiredcapacity := minOfInt(ac.getCardinality(), value2.getCardinality())
+	answer := newArrayContainerCapacity(desiredcapacity)
+	length := intersection2by2(
+		ac.content,
+		value2.content,
+		answer.content)
+	answer.content = answer.content[:length]
+	return answer
+}
+
+func (ac *arrayContainer) andArrayCardinality(value2 *arrayContainer) int {
+	return intersection2by2Cardinality(
+		ac.content,
+		value2.content)
+}
+
+func (ac *arrayContainer) intersectsArray(value2 *arrayContainer) bool {
+	return intersects2by2(
+		ac.content,
+		value2.content)
+}
+
+func (ac *arrayContainer) iandArray(value2 *arrayContainer) container {
+	length := intersection2by2(
+		ac.content,
+		value2.content,
+		ac.content)
+	ac.content = ac.content[:length]
+	return ac
+}
+
+func (ac *arrayContainer) getCardinality() int {
+	return len(ac.content)
+}
+
+func (ac *arrayContainer) rank(x uint16) int {
+	answer := binarySearch(ac.content, x)
+	if answer >= 0 {
+		return answer + 1
+	}
+	return -answer - 1
+
+}
+
+func (ac *arrayContainer) selectInt(x uint16) int {
+	return int(ac.content[x])
+}
+
+func (ac *arrayContainer) clone() container {
+	ptr := arrayContainer{make([]uint16, len(ac.content))}
+	copy(ptr.content, ac.content[:])
+	return &ptr
+}
+
+func (ac *arrayContainer) contains(x uint16) bool {
+	return binarySearch(ac.content, x) >= 0
+}
+
+func (ac *arrayContainer) loadData(bitmapContainer *bitmapContainer) {
+	ac.content = make([]uint16, bitmapContainer.cardinality, bitmapContainer.cardinality)
+	bitmapContainer.fillArray(ac.content)
+}
+func newArrayContainer() *arrayContainer {
+	p := new(arrayContainer)
+	return p
+}
+
+func newArrayContainerFromBitmap(bc *bitmapContainer) *arrayContainer {
+	ac := &arrayContainer{}
+	ac.loadData(bc)
+	return ac
+}
+
+func newArrayContainerCapacity(size int) *arrayContainer {
+	p := new(arrayContainer)
+	p.content = make([]uint16, 0, size)
+	return p
+}
+
+func newArrayContainerSize(size int) *arrayContainer {
+	p := new(arrayContainer)
+	p.content = make([]uint16, size, size)
+	return p
+}
+
+func newArrayContainerRange(firstOfRun, lastOfRun int) *arrayContainer {
+	valuesInRange := lastOfRun - firstOfRun + 1
+	this := newArrayContainerCapacity(valuesInRange)
+	for i := 0; i < valuesInRange; i++ {
+		this.content = append(this.content, uint16(firstOfRun+i))
+	}
+	return this
+}
+
+func (ac *arrayContainer) numberOfRuns() (nr int) {
+	n := len(ac.content)
+	var runlen uint16
+	var cur, prev uint16
+
+	switch n {
+	case 0:
+		return 0
+	case 1:
+		return 1
+	default:
+		for i := 1; i < n; i++ {
+			prev = ac.content[i-1]
+			cur = ac.content[i]
+
+			if cur == prev+1 {
+				runlen++
+			} else {
+				if cur < prev {
+					panic("then fundamental arrayContainer assumption of sorted ac.content was broken")
+				}
+				if cur == prev {
+					panic("then fundamental arrayContainer assumption of deduplicated content was broken")
+				} else {
+					nr++
+					runlen = 0
+				}
+			}
+		}
+		nr++
+	}
+	return
+}
+
+// convert to run or array *if needed*
+func (ac *arrayContainer) toEfficientContainer() container {
+
+	numRuns := ac.numberOfRuns()
+
+	sizeAsRunContainer := runContainer16SerializedSizeInBytes(numRuns)
+	sizeAsBitmapContainer := bitmapContainerSizeInBytes()
+	card := ac.getCardinality()
+	sizeAsArrayContainer := arrayContainerSizeInBytes(card)
+
+	if sizeAsRunContainer <= minOfInt(sizeAsBitmapContainer, sizeAsArrayContainer) {
+		return newRunContainer16FromArray(ac)
+	}
+	if card <= arrayDefaultMaxSize {
+		return ac
+	}
+	return ac.toBitmapContainer()
+}
+
+func (ac *arrayContainer) containerType() contype {
+	return arrayContype
+}
+
+func (ac *arrayContainer) addOffset(x uint16) []container {
+	low := &arrayContainer{}
+	high := &arrayContainer{}
+	for _, val := range ac.content {
+		y := uint32(val) + uint32(x)
+		if highbits(y) > 0 {
+			high.content = append(high.content, lowbits(y))
+		} else {
+			low.content = append(low.content, lowbits(y))
+		}
+	}
+	return []container{low, high}
+}

vendor/github.com/RoaringBitmap/roaring/arraycontainer_gen.go

@@ -0,0 +1,134 @@
+package roaring
+
+// NOTE: THIS FILE WAS PRODUCED BY THE
+// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp)
+// DO NOT EDIT
+
+import "github.com/tinylib/msgp/msgp"
+
+// Deprecated: DecodeMsg implements msgp.Decodable
+func (z *arrayContainer) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zbzg uint32
+	zbzg, err = dc.ReadMapHeader()
+	if err != nil {
+		return
+	}
+	for zbzg > 0 {
+		zbzg--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "content":
+			var zbai uint32
+			zbai, err = dc.ReadArrayHeader()
+			if err != nil {
+				return
+			}
+			if cap(z.content) >= int(zbai) {
+				z.content = (z.content)[:zbai]
+			} else {
+				z.content = make([]uint16, zbai)
+			}
+			for zxvk := range z.content {
+				z.content[zxvk], err = dc.ReadUint16()
+				if err != nil {
+					return
+				}
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				return
+			}
+		}
+	}
+	return
+}
+
+// Deprecated: EncodeMsg implements msgp.Encodable
+func (z *arrayContainer) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 1
+	// write "content"
+	err = en.Append(0x81, 0xa7, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74)
+	if err != nil {
+		return err
+	}
+	err = en.WriteArrayHeader(uint32(len(z.content)))
+	if err != nil {
+		return
+	}
+	for zxvk := range z.content {
+		err = en.WriteUint16(z.content[zxvk])
+		if err != nil {
+			return
+		}
+	}
+	return
+}
+
+// Deprecated: MarshalMsg implements msgp.Marshaler
+func (z *arrayContainer) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 1
+	// string "content"
+	o = append(o, 0x81, 0xa7, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74)
+	o = msgp.AppendArrayHeader(o, uint32(len(z.content)))
+	for zxvk := range z.content {
+		o = msgp.AppendUint16(o, z.content[zxvk])
+	}
+	return
+}
+
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
+func (z *arrayContainer) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zcmr uint32
+	zcmr, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		return
+	}
+	for zcmr > 0 {
+		zcmr--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "content":
+			var zajw uint32
+			zajw, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			if err != nil {
+				return
+			}
+			if cap(z.content) >= int(zajw) {
+				z.content = (z.content)[:zajw]
+			} else {
+				z.content = make([]uint16, zajw)
+			}
+			for zxvk := range z.content {
+				z.content[zxvk], bts, err = msgp.ReadUint16Bytes(bts)
+				if err != nil {
+					return
+				}
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *arrayContainer) Msgsize() (s int) {
+	s = 1 + 8 + msgp.ArrayHeaderSize + (len(z.content) * (msgp.Uint16Size))
+	return
+}

vendor/github.com/RoaringBitmap/roaring/bitmapcontainer_gen.go

@@ -0,0 +1,415 @@
+package roaring
+
+// NOTE: THIS FILE WAS PRODUCED BY THE
+// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp)
+// DO NOT EDIT
+
+import "github.com/tinylib/msgp/msgp"
+
+// Deprecated: DecodeMsg implements msgp.Decodable
+func (z *bitmapContainer) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zbzg uint32
+	zbzg, err = dc.ReadMapHeader()
+	if err != nil {
+		return
+	}
+	for zbzg > 0 {
+		zbzg--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "cardinality":
+			z.cardinality, err = dc.ReadInt()
+			if err != nil {
+				return
+			}
+		case "bitmap":
+			var zbai uint32
+			zbai, err = dc.ReadArrayHeader()
+			if err != nil {
+				return
+			}
+			if cap(z.bitmap) >= int(zbai) {
+				z.bitmap = (z.bitmap)[:zbai]
+			} else {
+				z.bitmap = make([]uint64, zbai)
+			}
+			for zxvk := range z.bitmap {
+				z.bitmap[zxvk], err = dc.ReadUint64()
+				if err != nil {
+					return
+				}
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				return
+			}
+		}
+	}
+	return
+}
+
+// Deprecated: EncodeMsg implements msgp.Encodable
+func (z *bitmapContainer) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 2
+	// write "cardinality"
+	err = en.Append(0x82, 0xab, 0x63, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x74, 0x79)
+	if err != nil {
+		return err
+	}
+	err = en.WriteInt(z.cardinality)
+	if err != nil {
+		return
+	}
+	// write "bitmap"
+	err = en.Append(0xa6, 0x62, 0x69, 0x74, 0x6d, 0x61, 0x70)
+	if err != nil {
+		return err
+	}
+	err = en.WriteArrayHeader(uint32(len(z.bitmap)))
+	if err != nil {
+		return
+	}
+	for zxvk := range z.bitmap {
+		err = en.WriteUint64(z.bitmap[zxvk])
+		if err != nil {
+			return
+		}
+	}
+	return
+}
+
+// Deprecated: MarshalMsg implements msgp.Marshaler
+func (z *bitmapContainer) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 2
+	// string "cardinality"
+	o = append(o, 0x82, 0xab, 0x63, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x74, 0x79)
+	o = msgp.AppendInt(o, z.cardinality)
+	// string "bitmap"
+	o = append(o, 0xa6, 0x62, 0x69, 0x74, 0x6d, 0x61, 0x70)
+	o = msgp.AppendArrayHeader(o, uint32(len(z.bitmap)))
+	for zxvk := range z.bitmap {
+		o = msgp.AppendUint64(o, z.bitmap[zxvk])
+	}
+	return
+}
+
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
+func (z *bitmapContainer) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zcmr uint32
+	zcmr, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		return
+	}
+	for zcmr > 0 {
+		zcmr--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "cardinality":
+			z.cardinality, bts, err = msgp.ReadIntBytes(bts)
+			if err != nil {
+				return
+			}
+		case "bitmap":
+			var zajw uint32
+			zajw, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			if err != nil {
+				return
+			}
+			if cap(z.bitmap) >= int(zajw) {
+				z.bitmap = (z.bitmap)[:zajw]
+			} else {
+				z.bitmap = make([]uint64, zajw)
+			}
+			for zxvk := range z.bitmap {
+				z.bitmap[zxvk], bts, err = msgp.ReadUint64Bytes(bts)
+				if err != nil {
+					return
+				}
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *bitmapContainer) Msgsize() (s int) {
+	s = 1 + 12 + msgp.IntSize + 7 + msgp.ArrayHeaderSize + (len(z.bitmap) * (msgp.Uint64Size))
+	return
+}
+
+// Deprecated: DecodeMsg implements msgp.Decodable
+func (z *bitmapContainerShortIterator) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zhct uint32
+	zhct, err = dc.ReadMapHeader()
+	if err != nil {
+		return
+	}
+	for zhct > 0 {
+		zhct--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "ptr":
+			if dc.IsNil() {
+				err = dc.ReadNil()
+				if err != nil {
+					return
+				}
+				z.ptr = nil
+			} else {
+				if z.ptr == nil {
+					z.ptr = new(bitmapContainer)
+				}
+				var zcua uint32
+				zcua, err = dc.ReadMapHeader()
+				if err != nil {
+					return
+				}
+				for zcua > 0 {
+					zcua--
+					field, err = dc.ReadMapKeyPtr()
+					if err != nil {
+						return
+					}
+					switch msgp.UnsafeString(field) {
+					case "cardinality":
+						z.ptr.cardinality, err = dc.ReadInt()
+						if err != nil {
+							return
+						}
+					case "bitmap":
+						var zxhx uint32
+						zxhx, err = dc.ReadArrayHeader()
+						if err != nil {
+							return
+						}
+						if cap(z.ptr.bitmap) >= int(zxhx) {
+							z.ptr.bitmap = (z.ptr.bitmap)[:zxhx]
+						} else {
+							z.ptr.bitmap = make([]uint64, zxhx)
+						}
+						for zwht := range z.ptr.bitmap {
+							z.ptr.bitmap[zwht], err = dc.ReadUint64()
+							if err != nil {
+								return
+							}
+						}
+					default:
+						err = dc.Skip()
+						if err != nil {
+							return
+						}
+					}
+				}
+			}
+		case "i":
+			z.i, err = dc.ReadInt()
+			if err != nil {
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				return
+			}
+		}
+	}
+	return
+}
+
+// Deprecated: EncodeMsg implements msgp.Encodable
+func (z *bitmapContainerShortIterator) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 2
+	// write "ptr"
+	err = en.Append(0x82, 0xa3, 0x70, 0x74, 0x72)
+	if err != nil {
+		return err
+	}
+	if z.ptr == nil {
+		err = en.WriteNil()
+		if err != nil {
+			return
+		}
+	} else {
+		// map header, size 2
+		// write "cardinality"
+		err = en.Append(0x82, 0xab, 0x63, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x74, 0x79)
+		if err != nil {
+			return err
+		}
+		err = en.WriteInt(z.ptr.cardinality)
+		if err != nil {
+			return
+		}
+		// write "bitmap"
+		err = en.Append(0xa6, 0x62, 0x69, 0x74, 0x6d, 0x61, 0x70)
+		if err != nil {
+			return err
+		}
+		err = en.WriteArrayHeader(uint32(len(z.ptr.bitmap)))
+		if err != nil {
+			return
+		}
+		for zwht := range z.ptr.bitmap {
+			err = en.WriteUint64(z.ptr.bitmap[zwht])
+			if err != nil {
+				return
+			}
+		}
+	}
+	// write "i"
+	err = en.Append(0xa1, 0x69)
+	if err != nil {
+		return err
+	}
+	err = en.WriteInt(z.i)
+	if err != nil {
+		return
+	}
+	return
+}
+
+// Deprecated: MarshalMsg implements msgp.Marshaler
+func (z *bitmapContainerShortIterator) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 2
+	// string "ptr"
+	o = append(o, 0x82, 0xa3, 0x70, 0x74, 0x72)
+	if z.ptr == nil {
+		o = msgp.AppendNil(o)
+	} else {
+		// map header, size 2
+		// string "cardinality"
+		o = append(o, 0x82, 0xab, 0x63, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x74, 0x79)
+		o = msgp.AppendInt(o, z.ptr.cardinality)
+		// string "bitmap"
+		o = append(o, 0xa6, 0x62, 0x69, 0x74, 0x6d, 0x61, 0x70)
+		o = msgp.AppendArrayHeader(o, uint32(len(z.ptr.bitmap)))
+		for zwht := range z.ptr.bitmap {
+			o = msgp.AppendUint64(o, z.ptr.bitmap[zwht])
+		}
+	}
+	// string "i"
+	o = append(o, 0xa1, 0x69)
+	o = msgp.AppendInt(o, z.i)
+	return
+}
+
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
+func (z *bitmapContainerShortIterator) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zlqf uint32
+	zlqf, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		return
+	}
+	for zlqf > 0 {
+		zlqf--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "ptr":
+			if msgp.IsNil(bts) {
+				bts, err = msgp.ReadNilBytes(bts)
+				if err != nil {
+					return
+				}
+				z.ptr = nil
+			} else {
+				if z.ptr == nil {
+					z.ptr = new(bitmapContainer)
+				}
+				var zdaf uint32
+				zdaf, bts, err = msgp.ReadMapHeaderBytes(bts)
+				if err != nil {
+					return
+				}
+				for zdaf > 0 {
+					zdaf--
+					field, bts, err = msgp.ReadMapKeyZC(bts)
+					if err != nil {
+						return
+					}
+					switch msgp.UnsafeString(field) {
+					case "cardinality":
+						z.ptr.cardinality, bts, err = msgp.ReadIntBytes(bts)
+						if err != nil {
+							return
+						}
+					case "bitmap":
+						var zpks uint32
+						zpks, bts, err = msgp.ReadArrayHeaderBytes(bts)
+						if err != nil {
+							return
+						}
+						if cap(z.ptr.bitmap) >= int(zpks) {
+							z.ptr.bitmap = (z.ptr.bitmap)[:zpks]
+						} else {
+							z.ptr.bitmap = make([]uint64, zpks)
+						}
+						for zwht := range z.ptr.bitmap {
+							z.ptr.bitmap[zwht], bts, err = msgp.ReadUint64Bytes(bts)
+							if err != nil {
+								return
+							}
+						}
+					default:
+						bts, err = msgp.Skip(bts)
+						if err != nil {
+							return
+						}
+					}
+				}
+			}
+		case "i":
+			z.i, bts, err = msgp.ReadIntBytes(bts)
+			if err != nil {
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *bitmapContainerShortIterator) Msgsize() (s int) {
+	s = 1 + 4
+	if z.ptr == nil {
+		s += msgp.NilSize
+	} else {
+		s += 1 + 12 + msgp.IntSize + 7 + msgp.ArrayHeaderSize + (len(z.ptr.bitmap) * (msgp.Uint64Size))
+	}
+	s += 2 + msgp.IntSize
+	return
+}

vendor/github.com/RoaringBitmap/roaring/byte_input.go

@@ -0,0 +1,161 @@
+package roaring
+
+import (
+	"encoding/binary"
+	"io"
+)
+
+type byteInput interface {
+	// next returns a slice containing the next n bytes from the buffer,
+	// advancing the buffer as if the bytes had been returned by Read.
+	next(n int) ([]byte, error)
+	// readUInt32 reads uint32 with LittleEndian order
+	readUInt32() (uint32, error)
+	// readUInt16 reads uint16 with LittleEndian order
+	readUInt16() (uint16, error)
+	// getReadBytes returns read bytes
+	getReadBytes() int64
+	// skipBytes skips exactly n bytes
+	skipBytes(n int) error
+}
+
+func newByteInputFromReader(reader io.Reader) byteInput {
+	return &byteInputAdapter{
+		r:         reader,
+		readBytes: 0,
+	}
+}
+
+func newByteInput(buf []byte) byteInput {
+	return &byteBuffer{
+		buf: buf,
+		off: 0,
+	}
+}
+
+type byteBuffer struct {
+	buf []byte
+	off int
+}
+
+// next returns a slice containing the next n bytes from the reader
+// If there are fewer bytes than the given n, io.ErrUnexpectedEOF will be returned
+func (b *byteBuffer) next(n int) ([]byte, error) {
+	m := len(b.buf) - b.off
+
+	if n > m {
+		return nil, io.ErrUnexpectedEOF
+	}
+
+	data := b.buf[b.off : b.off+n]
+	b.off += n
+
+	return data, nil
+}
+
+// readUInt32 reads uint32 with LittleEndian order
+func (b *byteBuffer) readUInt32() (uint32, error) {
+	if len(b.buf)-b.off < 4 {
+		return 0, io.ErrUnexpectedEOF
+	}
+
+	v := binary.LittleEndian.Uint32(b.buf[b.off:])
+	b.off += 4
+
+	return v, nil
+}
+
+// readUInt16 reads uint16 with LittleEndian order
+func (b *byteBuffer) readUInt16() (uint16, error) {
+	if len(b.buf)-b.off < 2 {
+		return 0, io.ErrUnexpectedEOF
+	}
+
+	v := binary.LittleEndian.Uint16(b.buf[b.off:])
+	b.off += 2
+
+	return v, nil
+}
+
+// getReadBytes returns read bytes
+func (b *byteBuffer) getReadBytes() int64 {
+	return int64(b.off)
+}
+
+// skipBytes skips exactly n bytes
+func (b *byteBuffer) skipBytes(n int) error {
+	m := len(b.buf) - b.off
+
+	if n > m {
+		return io.ErrUnexpectedEOF
+	}
+
+	b.off += n
+
+	return nil
+}
+
+// reset resets the given buffer with a new byte slice
+func (b *byteBuffer) reset(buf []byte) {
+	b.buf = buf
+	b.off = 0
+}
+
+type byteInputAdapter struct {
+	r         io.Reader
+	readBytes int
+}
+
+// next returns a slice containing the next n bytes from the buffer,
+// advancing the buffer as if the bytes had been returned by Read.
+func (b *byteInputAdapter) next(n int) ([]byte, error) {
+	buf := make([]byte, n)
+	m, err := io.ReadAtLeast(b.r, buf, n)
+	b.readBytes += m
+
+	if err != nil {
+		return nil, err
+	}
+
+	return buf, nil
+}
+
+// readUInt32 reads uint32 with LittleEndian order
+func (b *byteInputAdapter) readUInt32() (uint32, error) {
+	buf, err := b.next(4)
+
+	if err != nil {
+		return 0, err
+	}
+
+	return binary.LittleEndian.Uint32(buf), nil
+}
+
+// readUInt16 reads uint16 with LittleEndian order
+func (b *byteInputAdapter) readUInt16() (uint16, error) {
+	buf, err := b.next(2)
+
+	if err != nil {
+		return 0, err
+	}
+
+	return binary.LittleEndian.Uint16(buf), nil
+}
+
+// getReadBytes returns read bytes
+func (b *byteInputAdapter) getReadBytes() int64 {
+	return int64(b.readBytes)
+}
+
+// skipBytes skips exactly n bytes
+func (b *byteInputAdapter) skipBytes(n int) error {
+	_, err := b.next(n)
+
+	return err
+}
+
+// reset resets the given buffer with a new stream
+func (b *byteInputAdapter) reset(stream io.Reader) {
+	b.r = stream
+	b.readBytes = 0
+}

vendor/github.com/RoaringBitmap/roaring/clz.go

@@ -0,0 +1,11 @@
+// +build go1.9
+// "go1.9", from Go version 1.9 onward
+// See https://golang.org/pkg/go/build/#hdr-Build_Constraints
+
+package roaring
+
+import "math/bits"
+
+func countLeadingZeros(x uint64) int {
+	return bits.LeadingZeros64(x)
+}

vendor/github.com/RoaringBitmap/roaring/clz_compat.go

@@ -0,0 +1,36 @@
+// +build !go1.9
+
+package roaring
+
+// LeadingZeroBits returns the number of consecutive most significant zero
+// bits of x.
+func countLeadingZeros(i uint64) int {
+	if i == 0 {
+		return 64
+	}
+	n := 1
+	x := uint32(i >> 32)
+	if x == 0 {
+		n += 32
+		x = uint32(i)
+	}
+	if (x >> 16) == 0 {
+		n += 16
+		x <<= 16
+	}
+	if (x >> 24) == 0 {
+		n += 8
+		x <<= 8
+	}
+	if x>>28 == 0 {
+		n += 4
+		x <<= 4
+	}
+	if x>>30 == 0 {
+		n += 2
+		x <<= 2
+
+	}
+	n -= int(x >> 31)
+	return n
+}

vendor/github.com/RoaringBitmap/roaring/ctz.go

@@ -0,0 +1,11 @@
+// +build go1.9
+// "go1.9", from Go version 1.9 onward
+// See https://golang.org/pkg/go/build/#hdr-Build_Constraints
+
+package roaring
+
+import "math/bits"
+
+func countTrailingZeros(x uint64) int {
+	return bits.TrailingZeros64(x)
+}

vendor/github.com/RoaringBitmap/roaring/ctz_compat.go

@@ -0,0 +1,71 @@
+// +build !go1.9
+
+package roaring
+
+// Reuse of portions of go/src/math/big standard lib code
+// under this license:
+/*
+Copyright (c) 2009 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+const deBruijn32 = 0x077CB531
+
+var deBruijn32Lookup = []byte{
+	0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8,
+	31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9,
+}
+
+const deBruijn64 = 0x03f79d71b4ca8b09
+
+var deBruijn64Lookup = []byte{
+	0, 1, 56, 2, 57, 49, 28, 3, 61, 58, 42, 50, 38, 29, 17, 4,
+	62, 47, 59, 36, 45, 43, 51, 22, 53, 39, 33, 30, 24, 18, 12, 5,
+	63, 55, 48, 27, 60, 41, 37, 16, 46, 35, 44, 21, 52, 32, 23, 11,
+	54, 26, 40, 15, 34, 20, 31, 10, 25, 14, 19, 9, 13, 8, 7, 6,
+}
+
+// trailingZeroBits returns the number of consecutive least significant zero
+// bits of x.
+func countTrailingZeros(x uint64) int {
+	// x & -x leaves only the right-most bit set in the word. Let k be the
+	// index of that bit. Since only a single bit is set, the value is two
+	// to the power of k. Multiplying by a power of two is equivalent to
+	// left shifting, in this case by k bits. The de Bruijn constant is
+	// such that all six bit, consecutive substrings are distinct.
+	// Therefore, if we have a left shifted version of this constant we can
+	// find by how many bits it was shifted by looking at which six bit
+	// substring ended up at the top of the word.
+	// (Knuth, volume 4, section 7.3.1)
+	if x == 0 {
+		// We have to special case 0; the fomula
+		// below doesn't work for 0.
+		return 64
+	}
+	return int(deBruijn64Lookup[((x&-x)*(deBruijn64))>>58])
+}

vendor/github.com/RoaringBitmap/roaring/fastaggregation.go

@@ -0,0 +1,215 @@
+package roaring
+
+import (
+	"container/heap"
+)
+
+// Or function that requires repairAfterLazy
+func lazyOR(x1, x2 *Bitmap) *Bitmap {
+	answer := NewBitmap()
+	pos1 := 0
+	pos2 := 0
+	length1 := x1.highlowcontainer.size()
+	length2 := x2.highlowcontainer.size()
+main:
+	for (pos1 < length1) && (pos2 < length2) {
+		s1 := x1.highlowcontainer.getKeyAtIndex(pos1)
+		s2 := x2.highlowcontainer.getKeyAtIndex(pos2)
+
+		for {
+			if s1 < s2 {
+				answer.highlowcontainer.appendCopy(x1.highlowcontainer, pos1)
+				pos1++
+				if pos1 == length1 {
+					break main
+				}
+				s1 = x1.highlowcontainer.getKeyAtIndex(pos1)
+			} else if s1 > s2 {
+				answer.highlowcontainer.appendCopy(x2.highlowcontainer, pos2)
+				pos2++
+				if pos2 == length2 {
+					break main
+				}
+				s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
+			} else {
+				c1 := x1.highlowcontainer.getContainerAtIndex(pos1)
+				switch t := c1.(type) {
+				case *arrayContainer:
+					c1 = t.toBitmapContainer()
+				case *runContainer16:
+					if !t.isFull() {
+						c1 = t.toBitmapContainer()
+					}
+				}
+
+				answer.highlowcontainer.appendContainer(s1, c1.lazyOR(x2.highlowcontainer.getContainerAtIndex(pos2)), false)
+				pos1++
+				pos2++
+				if (pos1 == length1) || (pos2 == length2) {
+					break main
+				}
+				s1 = x1.highlowcontainer.getKeyAtIndex(pos1)
+				s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
+			}
+		}
+	}
+	if pos1 == length1 {
+		answer.highlowcontainer.appendCopyMany(x2.highlowcontainer, pos2, length2)
+	} else if pos2 == length2 {
+		answer.highlowcontainer.appendCopyMany(x1.highlowcontainer, pos1, length1)
+	}
+	return answer
+}
+
+// In-place Or function that requires repairAfterLazy
+func (x1 *Bitmap) lazyOR(x2 *Bitmap) *Bitmap {
+	pos1 := 0
+	pos2 := 0
+	length1 := x1.highlowcontainer.size()
+	length2 := x2.highlowcontainer.size()
+main:
+	for (pos1 < length1) && (pos2 < length2) {
+		s1 := x1.highlowcontainer.getKeyAtIndex(pos1)
+		s2 := x2.highlowcontainer.getKeyAtIndex(pos2)
+
+		for {
+			if s1 < s2 {
+				pos1++
+				if pos1 == length1 {
+					break main
+				}
+				s1 = x1.highlowcontainer.getKeyAtIndex(pos1)
+			} else if s1 > s2 {
+				x1.highlowcontainer.insertNewKeyValueAt(pos1, s2, x2.highlowcontainer.getContainerAtIndex(pos2).clone())
+				pos2++
+				pos1++
+				length1++
+				if pos2 == length2 {
+					break main
+				}
+				s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
+			} else {
+				c1 := x1.highlowcontainer.getContainerAtIndex(pos1)
+				switch t := c1.(type) {
+				case *arrayContainer:
+					c1 = t.toBitmapContainer()
+				case *runContainer16:
+					if !t.isFull() {
+						c1 = t.toBitmapContainer()
+					}
+				case *bitmapContainer:
+					c1 = x1.highlowcontainer.getWritableContainerAtIndex(pos1)
+				}
+
+				x1.highlowcontainer.containers[pos1] = c1.lazyIOR(x2.highlowcontainer.getContainerAtIndex(pos2))
+				x1.highlowcontainer.needCopyOnWrite[pos1] = false
+				pos1++
+				pos2++
+				if (pos1 == length1) || (pos2 == length2) {
+					break main
+				}
+				s1 = x1.highlowcontainer.getKeyAtIndex(pos1)
+				s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
+			}
+		}
+	}
+	if pos1 == length1 {
+		x1.highlowcontainer.appendCopyMany(x2.highlowcontainer, pos2, length2)
+	}
+	return x1
+}
+
+// to be called after lazy aggregates
+func (x1 *Bitmap) repairAfterLazy() {
+	for pos := 0; pos < x1.highlowcontainer.size(); pos++ {
+		c := x1.highlowcontainer.getContainerAtIndex(pos)
+		switch c.(type) {
+		case *bitmapContainer:
+			if c.(*bitmapContainer).cardinality == invalidCardinality {
+				c = x1.highlowcontainer.getWritableContainerAtIndex(pos)
+				c.(*bitmapContainer).computeCardinality()
+				if c.(*bitmapContainer).getCardinality() <= arrayDefaultMaxSize {
+					x1.highlowcontainer.setContainerAtIndex(pos, c.(*bitmapContainer).toArrayContainer())
+				} else if c.(*bitmapContainer).isFull() {
+					x1.highlowcontainer.setContainerAtIndex(pos, newRunContainer16Range(0, MaxUint16))
+				}
+			}
+		}
+	}
+}
+
+// FastAnd computes the intersection between many bitmaps quickly
+// Compared to the And function, it can take many bitmaps as input, thus saving the trouble
+// of manually calling "And" many times.
+func FastAnd(bitmaps ...*Bitmap) *Bitmap {
+	if len(bitmaps) == 0 {
+		return NewBitmap()
+	} else if len(bitmaps) == 1 {
+		return bitmaps[0].Clone()
+	}
+	answer := And(bitmaps[0], bitmaps[1])
+	for _, bm := range bitmaps[2:] {
+		answer.And(bm)
+	}
+	return answer
+}
+
+// FastOr computes the union between many bitmaps quickly, as opposed to having to call Or repeatedly.
+// It might also be faster than calling Or repeatedly.
+func FastOr(bitmaps ...*Bitmap) *Bitmap {
+	if len(bitmaps) == 0 {
+		return NewBitmap()
+	} else if len(bitmaps) == 1 {
+		return bitmaps[0].Clone()
+	}
+	answer := lazyOR(bitmaps[0], bitmaps[1])
+	for _, bm := range bitmaps[2:] {
+		answer = answer.lazyOR(bm)
+	}
+	// here is where repairAfterLazy is called.
+	answer.repairAfterLazy()
+	return answer
+}
+
+// HeapOr computes the union between many bitmaps quickly using a heap.
+// It might be faster than calling Or repeatedly.
+func HeapOr(bitmaps ...*Bitmap) *Bitmap {
+	if len(bitmaps) == 0 {
+		return NewBitmap()
+	}
+	// TODO:  for better speed, we could do the operation lazily, see Java implementation
+	pq := make(priorityQueue, len(bitmaps))
+	for i, bm := range bitmaps {
+		pq[i] = &item{bm, i}
+	}
+	heap.Init(&pq)
+
+	for pq.Len() > 1 {
+		x1 := heap.Pop(&pq).(*item)
+		x2 := heap.Pop(&pq).(*item)
+		heap.Push(&pq, &item{Or(x1.value, x2.value), 0})
+	}
+	return heap.Pop(&pq).(*item).value
+}
+
+// HeapXor computes the symmetric difference between many bitmaps quickly (as opposed to calling Xor repeated).
+// Internally, this function uses a heap.
+// It might be faster than calling Xor repeatedly.
+func HeapXor(bitmaps ...*Bitmap) *Bitmap {
+	if len(bitmaps) == 0 {
+		return NewBitmap()
+	}
+
+	pq := make(priorityQueue, len(bitmaps))
+	for i, bm := range bitmaps {
+		pq[i] = &item{bm, i}
+	}
+	heap.Init(&pq)
+
+	for pq.Len() > 1 {
+		x1 := heap.Pop(&pq).(*item)
+		x2 := heap.Pop(&pq).(*item)
+		heap.Push(&pq, &item{Xor(x1.value, x2.value), 0})
+	}
+	return heap.Pop(&pq).(*item).value
+}

vendor/github.com/RoaringBitmap/roaring/go.mod

@@ -0,0 +1,16 @@
+module github.com/RoaringBitmap/roaring
+
+go 1.12
+
+require (
+	github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2
+	github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 // indirect
+	github.com/golang/snappy v0.0.1 // indirect
+	github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99 // indirect
+	github.com/jtolds/gls v4.20.0+incompatible // indirect
+	github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae
+	github.com/philhofer/fwd v1.0.0 // indirect
+	github.com/stretchr/testify v1.4.0
+	github.com/tinylib/msgp v1.1.0
+	github.com/willf/bitset v1.1.10
+)

vendor/github.com/RoaringBitmap/roaring/go.sum

@@ -0,0 +1,30 @@
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2 h1:Ujru1hufTHVb++eG6OuNDKMxZnGIvF6o/u8q/8h2+I4=
+github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
+github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 h1:gclg6gY70GLy3PbkQ1AERPfmLMMagS60DKF78eWwLn8=
+github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99 h1:twflg0XRTjwKpxb/jFExr4HGq6on2dEOmnL6FV+fgPw=
+github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae h1:VeRdUYdCw49yizlSbMEn2SZ+gT+3IUKx8BqxyQdz+BY=
+github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
+github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ=
+github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU=
+github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
+github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc=
+github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

vendor/github.com/RoaringBitmap/roaring/manyiterator.go

@@ -0,0 +1,18 @@
+package roaring
+
+type manyIterable interface {
+	nextMany(hs uint32, buf []uint32) int
+}
+
+func (si *shortIterator) nextMany(hs uint32, buf []uint32) int {
+	n := 0
+	l := si.loc
+	s := si.slice
+	for n < len(buf) && l < len(s) {
+		buf[n] = uint32(s[l]) | hs
+		l++
+		n++
+	}
+	si.loc = l
+	return n
+}

vendor/github.com/RoaringBitmap/roaring/parallel.go

@@ -0,0 +1,613 @@
+package roaring
+
+import (
+	"container/heap"
+	"fmt"
+	"runtime"
+	"sync"
+)
+
+var defaultWorkerCount = runtime.NumCPU()
+
+type bitmapContainerKey struct {
+	key    uint16
+	idx    int
+	bitmap *Bitmap
+}
+
+type multipleContainers struct {
+	key        uint16
+	containers []container
+	idx        int
+}
+
+type keyedContainer struct {
+	key       uint16
+	container container
+	idx       int
+}
+
+type bitmapContainerHeap []bitmapContainerKey
+
+func (h bitmapContainerHeap) Len() int           { return len(h) }
+func (h bitmapContainerHeap) Less(i, j int) bool { return h[i].key < h[j].key }
+func (h bitmapContainerHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }
+
+func (h *bitmapContainerHeap) Push(x interface{}) {
+	// Push and Pop use pointer receivers because they modify the slice's length,
+	// not just its contents.
+	*h = append(*h, x.(bitmapContainerKey))
+}
+
+func (h *bitmapContainerHeap) Pop() interface{} {
+	old := *h
+	n := len(old)
+	x := old[n-1]
+	*h = old[0 : n-1]
+	return x
+}
+
+func (h bitmapContainerHeap) Peek() bitmapContainerKey {
+	return h[0]
+}
+
+func (h *bitmapContainerHeap) popIncrementing() (key uint16, container container) {
+	k := h.Peek()
+	key = k.key
+	container = k.bitmap.highlowcontainer.containers[k.idx]
+
+	newIdx := k.idx + 1
+	if newIdx < k.bitmap.highlowcontainer.size() {
+		k = bitmapContainerKey{
+			k.bitmap.highlowcontainer.keys[newIdx],
+			newIdx,
+			k.bitmap,
+		}
+		(*h)[0] = k
+		heap.Fix(h, 0)
+	} else {
+		heap.Pop(h)
+	}
+
+	return
+}
+
+func (h *bitmapContainerHeap) Next(containers []container) multipleContainers {
+	if h.Len() == 0 {
+		return multipleContainers{}
+	}
+
+	key, container := h.popIncrementing()
+	containers = append(containers, container)
+
+	for h.Len() > 0 && key == h.Peek().key {
+		_, container = h.popIncrementing()
+		containers = append(containers, container)
+	}
+
+	return multipleContainers{
+		key,
+		containers,
+		-1,
+	}
+}
+
+func newBitmapContainerHeap(bitmaps ...*Bitmap) bitmapContainerHeap {
+	// Initialize heap
+	var h bitmapContainerHeap = make([]bitmapContainerKey, 0, len(bitmaps))
+	for _, bitmap := range bitmaps {
+		if !bitmap.IsEmpty() {
+			key := bitmapContainerKey{
+				bitmap.highlowcontainer.keys[0],
+				0,
+				bitmap,
+			}
+			h = append(h, key)
+		}
+	}
+
+	heap.Init(&h)
+
+	return h
+}
+
+func repairAfterLazy(c container) container {
+	switch t := c.(type) {
+	case *bitmapContainer:
+		if t.cardinality == invalidCardinality {
+			t.computeCardinality()
+		}
+
+		if t.getCardinality() <= arrayDefaultMaxSize {
+			return t.toArrayContainer()
+		} else if c.(*bitmapContainer).isFull() {
+			return newRunContainer16Range(0, MaxUint16)
+		}
+	}
+
+	return c
+}
+
+func toBitmapContainer(c container) container {
+	switch t := c.(type) {
+	case *arrayContainer:
+		return t.toBitmapContainer()
+	case *runContainer16:
+		if !t.isFull() {
+			return t.toBitmapContainer()
+		}
+	}
+	return c
+}
+
+func appenderRoutine(bitmapChan chan<- *Bitmap, resultChan <-chan keyedContainer, expectedKeysChan <-chan int) {
+	expectedKeys := -1
+	appendedKeys := 0
+	var keys []uint16
+	var containers []container
+	for appendedKeys != expectedKeys {
+		select {
+		case item := <-resultChan:
+			if len(keys) <= item.idx {
+				keys = append(keys, make([]uint16, item.idx-len(keys)+1)...)
+				containers = append(containers, make([]container, item.idx-len(containers)+1)...)
+			}
+			keys[item.idx] = item.key
+			containers[item.idx] = item.container
+
+			appendedKeys++
+		case msg := <-expectedKeysChan:
+			expectedKeys = msg
+		}
+	}
+	answer := &Bitmap{
+		roaringArray{
+			make([]uint16, 0, expectedKeys),
+			make([]container, 0, expectedKeys),
+			make([]bool, 0, expectedKeys),
+			false,
+			nil,
+		},
+	}
+	for i := range keys {
+		if containers[i] != nil { // in case a resulting container was empty, see ParAnd function
+			answer.highlowcontainer.appendContainer(keys[i], containers[i], false)
+		}
+	}
+
+	bitmapChan <- answer
+}
+
+// ParHeapOr computes the union (OR) of all provided bitmaps in parallel,
+// where the parameter "parallelism" determines how many workers are to be used
+// (if it is set to 0, a default number of workers is chosen)
+// ParHeapOr uses a heap to compute the union. For rare cases it might be faster than ParOr
+func ParHeapOr(parallelism int, bitmaps ...*Bitmap) *Bitmap {
+
+	bitmapCount := len(bitmaps)
+	if bitmapCount == 0 {
+		return NewBitmap()
+	} else if bitmapCount == 1 {
+		return bitmaps[0].Clone()
+	}
+
+	if parallelism == 0 {
+		parallelism = defaultWorkerCount
+	}
+
+	h := newBitmapContainerHeap(bitmaps...)
+
+	bitmapChan := make(chan *Bitmap)
+	inputChan := make(chan multipleContainers, 128)
+	resultChan := make(chan keyedContainer, 32)
+	expectedKeysChan := make(chan int)
+
+	pool := sync.Pool{
+		New: func() interface{} {
+			return make([]container, 0, len(bitmaps))
+		},
+	}
+
+	orFunc := func() {
+		// Assumes only structs with >=2 containers are passed
+		for input := range inputChan {
+			c := toBitmapContainer(input.containers[0]).lazyOR(input.containers[1])
+			for _, next := range input.containers[2:] {
+				c = c.lazyIOR(next)
+			}
+			c = repairAfterLazy(c)
+			kx := keyedContainer{
+				input.key,
+				c,
+				input.idx,
+			}
+			resultChan <- kx
+			pool.Put(input.containers[:0])
+		}
+	}
+
+	go appenderRoutine(bitmapChan, resultChan, expectedKeysChan)
+
+	for i := 0; i < parallelism; i++ {
+		go orFunc()
+	}
+
+	idx := 0
+	for h.Len() > 0 {
+		ck := h.Next(pool.Get().([]container))
+		if len(ck.containers) == 1 {
+			resultChan <- keyedContainer{
+				ck.key,
+				ck.containers[0],
+				idx,
+			}
+			pool.Put(ck.containers[:0])
+		} else {
+			ck.idx = idx
+			inputChan <- ck
+		}
+		idx++
+	}
+	expectedKeysChan <- idx
+
+	bitmap := <-bitmapChan
+
+	close(inputChan)
+	close(resultChan)
+	close(expectedKeysChan)
+
+	return bitmap
+}
+
+// ParAnd computes the intersection (AND) of all provided bitmaps in parallel,
+// where the parameter "parallelism" determines how many workers are to be used
+// (if it is set to 0, a default number of workers is chosen)
+func ParAnd(parallelism int, bitmaps ...*Bitmap) *Bitmap {
+	bitmapCount := len(bitmaps)
+	if bitmapCount == 0 {
+		return NewBitmap()
+	} else if bitmapCount == 1 {
+		return bitmaps[0].Clone()
+	}
+
+	if parallelism == 0 {
+		parallelism = defaultWorkerCount
+	}
+
+	h := newBitmapContainerHeap(bitmaps...)
+
+	bitmapChan := make(chan *Bitmap)
+	inputChan := make(chan multipleContainers, 128)
+	resultChan := make(chan keyedContainer, 32)
+	expectedKeysChan := make(chan int)
+
+	andFunc := func() {
+		// Assumes only structs with >=2 containers are passed
+		for input := range inputChan {
+			c := input.containers[0].and(input.containers[1])
+			for _, next := range input.containers[2:] {
+				if c.getCardinality() == 0 {
+					break
+				}
+				c = c.iand(next)
+			}
+
+			// Send a nil explicitly if the result of the intersection is an empty container
+			if c.getCardinality() == 0 {
+				c = nil
+			}
+
+			kx := keyedContainer{
+				input.key,
+				c,
+				input.idx,
+			}
+			resultChan <- kx
+		}
+	}
+
+	go appenderRoutine(bitmapChan, resultChan, expectedKeysChan)
+
+	for i := 0; i < parallelism; i++ {
+		go andFunc()
+	}
+
+	idx := 0
+	for h.Len() > 0 {
+		ck := h.Next(make([]container, 0, 4))
+		if len(ck.containers) == bitmapCount {
+			ck.idx = idx
+			inputChan <- ck
+			idx++
+		}
+	}
+	expectedKeysChan <- idx
+
+	bitmap := <-bitmapChan
+
+	close(inputChan)
+	close(resultChan)
+	close(expectedKeysChan)
+
+	return bitmap
+}
+
+// ParOr computes the union (OR) of all provided bitmaps in parallel,
+// where the parameter "parallelism" determines how many workers are to be used
+// (if it is set to 0, a default number of workers is chosen)
+func ParOr(parallelism int, bitmaps ...*Bitmap) *Bitmap {
+	var lKey uint16 = MaxUint16
+	var hKey uint16
+
+	bitmapsFiltered := bitmaps[:0]
+	for _, b := range bitmaps {
+		if !b.IsEmpty() {
+			bitmapsFiltered = append(bitmapsFiltered, b)
+		}
+	}
+	bitmaps = bitmapsFiltered
+
+	for _, b := range bitmaps {
+		lKey = minOfUint16(lKey, b.highlowcontainer.keys[0])
+		hKey = maxOfUint16(hKey, b.highlowcontainer.keys[b.highlowcontainer.size()-1])
+	}
+
+	if lKey == MaxUint16 && hKey == 0 {
+		return New()
+	} else if len(bitmaps) == 1 {
+		return bitmaps[0]
+	}
+
+	keyRange := hKey - lKey + 1
+	if keyRange == 1 {
+		// revert to FastOr. Since the key range is 0
+		// no container-level aggregation parallelism is achievable
+		return FastOr(bitmaps...)
+	}
+
+	if parallelism == 0 {
+		parallelism = defaultWorkerCount
+	}
+
+	var chunkSize int
+	var chunkCount int
+	if parallelism*4 > int(keyRange) {
+		chunkSize = 1
+		chunkCount = int(keyRange)
+	} else {
+		chunkCount = parallelism * 4
+		chunkSize = (int(keyRange) + chunkCount - 1) / chunkCount
+	}
+
+	if chunkCount*chunkSize < int(keyRange) {
+		// it's fine to panic to indicate an implementation error
+		panic(fmt.Sprintf("invariant check failed: chunkCount * chunkSize < keyRange, %d * %d < %d", chunkCount, chunkSize, keyRange))
+	}
+
+	chunks := make([]*roaringArray, chunkCount)
+
+	chunkSpecChan := make(chan parChunkSpec, minOfInt(maxOfInt(64, 2*parallelism), int(chunkCount)))
+	chunkChan := make(chan parChunk, minOfInt(32, int(chunkCount)))
+
+	orFunc := func() {
+		for spec := range chunkSpecChan {
+			ra := lazyOrOnRange(&bitmaps[0].highlowcontainer, &bitmaps[1].highlowcontainer, spec.start, spec.end)
+			for _, b := range bitmaps[2:] {
+				ra = lazyIOrOnRange(ra, &b.highlowcontainer, spec.start, spec.end)
+			}
+
+			for i, c := range ra.containers {
+				ra.containers[i] = repairAfterLazy(c)
+			}
+
+			chunkChan <- parChunk{ra, spec.idx}
+		}
+	}
+
+	for i := 0; i < parallelism; i++ {
+		go orFunc()
+	}
+
+	go func() {
+		for i := 0; i < chunkCount; i++ {
+			spec := parChunkSpec{
+				start: uint16(int(lKey) + i*chunkSize),
+				end:   uint16(minOfInt(int(lKey)+(i+1)*chunkSize-1, int(hKey))),
+				idx:   int(i),
+			}
+			chunkSpecChan <- spec
+		}
+	}()
+
+	chunksRemaining := chunkCount
+	for chunk := range chunkChan {
+		chunks[chunk.idx] = chunk.ra
+		chunksRemaining--
+		if chunksRemaining == 0 {
+			break
+		}
+	}
+	close(chunkChan)
+	close(chunkSpecChan)
+
+	containerCount := 0
+	for _, chunk := range chunks {
+		containerCount += chunk.size()
+	}
+
+	result := Bitmap{
+		roaringArray{
+			containers:      make([]container, containerCount),
+			keys:            make([]uint16, containerCount),
+			needCopyOnWrite: make([]bool, containerCount),
+		},
+	}
+
+	resultOffset := 0
+	for _, chunk := range chunks {
+		copy(result.highlowcontainer.containers[resultOffset:], chunk.containers)
+		copy(result.highlowcontainer.keys[resultOffset:], chunk.keys)
+		copy(result.highlowcontainer.needCopyOnWrite[resultOffset:], chunk.needCopyOnWrite)
+		resultOffset += chunk.size()
+	}
+
+	return &result
+}
+
+type parChunkSpec struct {
+	start uint16
+	end   uint16
+	idx   int
+}
+
+type parChunk struct {
+	ra  *roaringArray
+	idx int
+}
+
+func (c parChunk) size() int {
+	return c.ra.size()
+}
+
+func parNaiveStartAt(ra *roaringArray, start uint16, last uint16) int {
+	for idx, key := range ra.keys {
+		if key >= start && key <= last {
+			return idx
+		} else if key > last {
+			break
+		}
+	}
+	return ra.size()
+}
+
+func lazyOrOnRange(ra1, ra2 *roaringArray, start, last uint16) *roaringArray {
+	answer := newRoaringArray()
+	length1 := ra1.size()
+	length2 := ra2.size()
+
+	idx1 := parNaiveStartAt(ra1, start, last)
+	idx2 := parNaiveStartAt(ra2, start, last)
+
+	var key1 uint16
+	var key2 uint16
+	if idx1 < length1 && idx2 < length2 {
+		key1 = ra1.getKeyAtIndex(idx1)
+		key2 = ra2.getKeyAtIndex(idx2)
+
+		for key1 <= last && key2 <= last {
+
+			if key1 < key2 {
+				answer.appendCopy(*ra1, idx1)
+				idx1++
+				if idx1 == length1 {
+					break
+				}
+				key1 = ra1.getKeyAtIndex(idx1)
+			} else if key1 > key2 {
+				answer.appendCopy(*ra2, idx2)
+				idx2++
+				if idx2 == length2 {
+					break
+				}
+				key2 = ra2.getKeyAtIndex(idx2)
+			} else {
+				c1 := ra1.getFastContainerAtIndex(idx1, false)
+
+				answer.appendContainer(key1, c1.lazyOR(ra2.getContainerAtIndex(idx2)), false)
+				idx1++
+				idx2++
+				if idx1 == length1 || idx2 == length2 {
+					break
+				}
+
+				key1 = ra1.getKeyAtIndex(idx1)
+				key2 = ra2.getKeyAtIndex(idx2)
+			}
+		}
+	}
+
+	if idx2 < length2 {
+		key2 = ra2.getKeyAtIndex(idx2)
+		for key2 <= last {
+			answer.appendCopy(*ra2, idx2)
+			idx2++
+			if idx2 == length2 {
+				break
+			}
+			key2 = ra2.getKeyAtIndex(idx2)
+		}
+	}
+
+	if idx1 < length1 {
+		key1 = ra1.getKeyAtIndex(idx1)
+		for key1 <= last {
+			answer.appendCopy(*ra1, idx1)
+			idx1++
+			if idx1 == length1 {
+				break
+			}
+			key1 = ra1.getKeyAtIndex(idx1)
+		}
+	}
+	return answer
+}
+
+func lazyIOrOnRange(ra1, ra2 *roaringArray, start, last uint16) *roaringArray {
+	length1 := ra1.size()
+	length2 := ra2.size()
+
+	idx1 := 0
+	idx2 := parNaiveStartAt(ra2, start, last)
+
+	var key1 uint16
+	var key2 uint16
+	if idx1 < length1 && idx2 < length2 {
+		key1 = ra1.getKeyAtIndex(idx1)
+		key2 = ra2.getKeyAtIndex(idx2)
+
+		for key1 <= last && key2 <= last {
+			if key1 < key2 {
+				idx1++
+				if idx1 >= length1 {
+					break
+				}
+				key1 = ra1.getKeyAtIndex(idx1)
+			} else if key1 > key2 {
+				ra1.insertNewKeyValueAt(idx1, key2, ra2.getContainerAtIndex(idx2))
+				ra1.needCopyOnWrite[idx1] = true
+				idx2++
+				idx1++
+				length1++
+				if idx2 >= length2 {
+					break
+				}
+				key2 = ra2.getKeyAtIndex(idx2)
+			} else {
+				c1 := ra1.getFastContainerAtIndex(idx1, true)
+
+				ra1.containers[idx1] = c1.lazyIOR(ra2.getContainerAtIndex(idx2))
+				ra1.needCopyOnWrite[idx1] = false
+				idx1++
+				idx2++
+				if idx1 >= length1 || idx2 >= length2 {
+					break
+				}
+
+				key1 = ra1.getKeyAtIndex(idx1)
+				key2 = ra2.getKeyAtIndex(idx2)
+			}
+		}
+	}
+	if idx2 < length2 {
+		key2 = ra2.getKeyAtIndex(idx2)
+		for key2 <= last {
+			ra1.appendCopy(*ra2, idx2)
+			idx2++
+			if idx2 >= length2 {
+				break
+			}
+			key2 = ra2.getKeyAtIndex(idx2)
+		}
+	}
+	return ra1
+}

vendor/github.com/RoaringBitmap/roaring/popcnt.go

@@ -0,0 +1,11 @@
+// +build go1.9
+// "go1.9", from Go version 1.9 onward
+// See https://golang.org/pkg/go/build/#hdr-Build_Constraints
+
+package roaring
+
+import "math/bits"
+
+func popcount(x uint64) uint64 {
+	return uint64(bits.OnesCount64(x))
+}

vendor/github.com/RoaringBitmap/roaring/popcnt_amd64.s

@@ -0,0 +1,103 @@
+// +build amd64,!appengine,!go1.9
+
+TEXT ·hasAsm(SB),4,$0-1
+MOVQ $1, AX
+CPUID
+SHRQ $23, CX
+ANDQ $1, CX
+MOVB CX, ret+0(FP)
+RET
+
+#define POPCNTQ_DX_DX BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0xd2
+
+TEXT ·popcntSliceAsm(SB),4,$0-32
+XORQ	AX, AX
+MOVQ	s+0(FP), SI
+MOVQ	s_len+8(FP), CX
+TESTQ	CX, CX
+JZ		popcntSliceEnd
+popcntSliceLoop:
+BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0x16 // POPCNTQ (SI), DX
+ADDQ	DX, AX
+ADDQ	$8, SI
+LOOP	popcntSliceLoop
+popcntSliceEnd:
+MOVQ	AX, ret+24(FP)
+RET
+
+TEXT ·popcntMaskSliceAsm(SB),4,$0-56
+XORQ	AX, AX
+MOVQ	s+0(FP), SI
+MOVQ	s_len+8(FP), CX
+TESTQ	CX, CX
+JZ		popcntMaskSliceEnd
+MOVQ	m+24(FP), DI
+popcntMaskSliceLoop:
+MOVQ	(DI), DX
+NOTQ	DX
+ANDQ	(SI), DX
+POPCNTQ_DX_DX
+ADDQ	DX, AX
+ADDQ	$8, SI
+ADDQ	$8, DI
+LOOP	popcntMaskSliceLoop
+popcntMaskSliceEnd:
+MOVQ	AX, ret+48(FP)
+RET
+
+TEXT ·popcntAndSliceAsm(SB),4,$0-56
+XORQ	AX, AX
+MOVQ	s+0(FP), SI
+MOVQ	s_len+8(FP), CX
+TESTQ	CX, CX
+JZ		popcntAndSliceEnd
+MOVQ	m+24(FP), DI
+popcntAndSliceLoop:
+MOVQ	(DI), DX
+ANDQ	(SI), DX
+POPCNTQ_DX_DX
+ADDQ	DX, AX
+ADDQ	$8, SI
+ADDQ	$8, DI
+LOOP	popcntAndSliceLoop
+popcntAndSliceEnd:
+MOVQ	AX, ret+48(FP)
+RET
+
+TEXT ·popcntOrSliceAsm(SB),4,$0-56
+XORQ	AX, AX
+MOVQ	s+0(FP), SI
+MOVQ	s_len+8(FP), CX
+TESTQ	CX, CX
+JZ		popcntOrSliceEnd
+MOVQ	m+24(FP), DI
+popcntOrSliceLoop:
+MOVQ	(DI), DX
+ORQ		(SI), DX
+POPCNTQ_DX_DX
+ADDQ	DX, AX
+ADDQ	$8, SI
+ADDQ	$8, DI
+LOOP	popcntOrSliceLoop
+popcntOrSliceEnd:
+MOVQ	AX, ret+48(FP)
+RET
+
+TEXT ·popcntXorSliceAsm(SB),4,$0-56
+XORQ	AX, AX
+MOVQ	s+0(FP), SI
+MOVQ	s_len+8(FP), CX
+TESTQ	CX, CX
+JZ		popcntXorSliceEnd
+MOVQ	m+24(FP), DI
+popcntXorSliceLoop:
+MOVQ	(DI), DX
+XORQ	(SI), DX
+POPCNTQ_DX_DX
+ADDQ	DX, AX
+ADDQ	$8, SI
+ADDQ	$8, DI
+LOOP	popcntXorSliceLoop
+popcntXorSliceEnd:
+MOVQ	AX, ret+48(FP)
+RET

vendor/github.com/RoaringBitmap/roaring/popcnt_asm.go

@@ -0,0 +1,67 @@
+// +build amd64,!appengine,!go1.9
+
+package roaring
+
+// *** the following functions are defined in popcnt_amd64.s
+
+//go:noescape
+
+func hasAsm() bool
+
+// useAsm is a flag used to select the GO or ASM implementation of the popcnt function
+var useAsm = hasAsm()
+
+//go:noescape
+
+func popcntSliceAsm(s []uint64) uint64
+
+//go:noescape
+
+func popcntMaskSliceAsm(s, m []uint64) uint64
+
+//go:noescape
+
+func popcntAndSliceAsm(s, m []uint64) uint64
+
+//go:noescape
+
+func popcntOrSliceAsm(s, m []uint64) uint64
+
+//go:noescape
+
+func popcntXorSliceAsm(s, m []uint64) uint64
+
+func popcntSlice(s []uint64) uint64 {
+	if useAsm {
+		return popcntSliceAsm(s)
+	}
+	return popcntSliceGo(s)
+}
+
+func popcntMaskSlice(s, m []uint64) uint64 {
+	if useAsm {
+		return popcntMaskSliceAsm(s, m)
+	}
+	return popcntMaskSliceGo(s, m)
+}
+
+func popcntAndSlice(s, m []uint64) uint64 {
+	if useAsm {
+		return popcntAndSliceAsm(s, m)
+	}
+	return popcntAndSliceGo(s, m)
+}
+
+func popcntOrSlice(s, m []uint64) uint64 {
+	if useAsm {
+		return popcntOrSliceAsm(s, m)
+	}
+	return popcntOrSliceGo(s, m)
+}
+
+func popcntXorSlice(s, m []uint64) uint64 {
+	if useAsm {
+		return popcntXorSliceAsm(s, m)
+	}
+	return popcntXorSliceGo(s, m)
+}

vendor/github.com/RoaringBitmap/roaring/popcnt_compat.go

@@ -0,0 +1,17 @@
+// +build !go1.9
+
+package roaring
+
+// bit population count, take from
+// https://code.google.com/p/go/issues/detail?id=4988#c11
+// credit: https://code.google.com/u/arnehormann/
+// credit: https://play.golang.org/p/U7SogJ7psJ
+// credit: http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
+func popcount(x uint64) uint64 {
+	x -= (x >> 1) & 0x5555555555555555
+	x = (x>>2)&0x3333333333333333 + x&0x3333333333333333
+	x += x >> 4
+	x &= 0x0f0f0f0f0f0f0f0f
+	x *= 0x0101010101010101
+	return x >> 56
+}

vendor/github.com/RoaringBitmap/roaring/popcnt_generic.go

@@ -0,0 +1,23 @@
+// +build !amd64 appengine go1.9
+
+package roaring
+
+func popcntSlice(s []uint64) uint64 {
+	return popcntSliceGo(s)
+}
+
+func popcntMaskSlice(s, m []uint64) uint64 {
+	return popcntMaskSliceGo(s, m)
+}
+
+func popcntAndSlice(s, m []uint64) uint64 {
+	return popcntAndSliceGo(s, m)
+}
+
+func popcntOrSlice(s, m []uint64) uint64 {
+	return popcntOrSliceGo(s, m)
+}
+
+func popcntXorSlice(s, m []uint64) uint64 {
+	return popcntXorSliceGo(s, m)
+}

vendor/github.com/RoaringBitmap/roaring/popcnt_slices.go

@@ -0,0 +1,41 @@
+package roaring
+
+func popcntSliceGo(s []uint64) uint64 {
+	cnt := uint64(0)
+	for _, x := range s {
+		cnt += popcount(x)
+	}
+	return cnt
+}
+
+func popcntMaskSliceGo(s, m []uint64) uint64 {
+	cnt := uint64(0)
+	for i := range s {
+		cnt += popcount(s[i] &^ m[i])
+	}
+	return cnt
+}
+
+func popcntAndSliceGo(s, m []uint64) uint64 {
+	cnt := uint64(0)
+	for i := range s {
+		cnt += popcount(s[i] & m[i])
+	}
+	return cnt
+}
+
+func popcntOrSliceGo(s, m []uint64) uint64 {
+	cnt := uint64(0)
+	for i := range s {
+		cnt += popcount(s[i] | m[i])
+	}
+	return cnt
+}
+
+func popcntXorSliceGo(s, m []uint64) uint64 {
+	cnt := uint64(0)
+	for i := range s {
+		cnt += popcount(s[i] ^ m[i])
+	}
+	return cnt
+}

vendor/github.com/RoaringBitmap/roaring/priorityqueue.go

@@ -0,0 +1,101 @@
+package roaring
+
+import "container/heap"
+
+/////////////
+// The priorityQueue is used to keep Bitmaps sorted.
+////////////
+
+type item struct {
+	value *Bitmap
+	index int
+}
+
+type priorityQueue []*item
+
+func (pq priorityQueue) Len() int { return len(pq) }
+
+func (pq priorityQueue) Less(i, j int) bool {
+	return pq[i].value.GetSizeInBytes() < pq[j].value.GetSizeInBytes()
+}
+
+func (pq priorityQueue) Swap(i, j int) {
+	pq[i], pq[j] = pq[j], pq[i]
+	pq[i].index = i
+	pq[j].index = j
+}
+
+func (pq *priorityQueue) Push(x interface{}) {
+	n := len(*pq)
+	item := x.(*item)
+	item.index = n
+	*pq = append(*pq, item)
+}
+
+func (pq *priorityQueue) Pop() interface{} {
+	old := *pq
+	n := len(old)
+	item := old[n-1]
+	item.index = -1 // for safety
+	*pq = old[0 : n-1]
+	return item
+}
+
+func (pq *priorityQueue) update(item *item, value *Bitmap) {
+	item.value = value
+	heap.Fix(pq, item.index)
+}
+
+/////////////
+// The containerPriorityQueue is used to keep the containers of various Bitmaps sorted.
+////////////
+
+type containeritem struct {
+	value    *Bitmap
+	keyindex int
+	index    int
+}
+
+type containerPriorityQueue []*containeritem
+
+func (pq containerPriorityQueue) Len() int { return len(pq) }
+
+func (pq containerPriorityQueue) Less(i, j int) bool {
+	k1 := pq[i].value.highlowcontainer.getKeyAtIndex(pq[i].keyindex)
+	k2 := pq[j].value.highlowcontainer.getKeyAtIndex(pq[j].keyindex)
+	if k1 != k2 {
+		return k1 < k2
+	}
+	c1 := pq[i].value.highlowcontainer.getContainerAtIndex(pq[i].keyindex)
+	c2 := pq[j].value.highlowcontainer.getContainerAtIndex(pq[j].keyindex)
+
+	return c1.getCardinality() > c2.getCardinality()
+}
+
+func (pq containerPriorityQueue) Swap(i, j int) {
+	pq[i], pq[j] = pq[j], pq[i]
+	pq[i].index = i
+	pq[j].index = j
+}
+
+func (pq *containerPriorityQueue) Push(x interface{}) {
+	n := len(*pq)
+	item := x.(*containeritem)
+	item.index = n
+	*pq = append(*pq, item)
+}
+
+func (pq *containerPriorityQueue) Pop() interface{} {
+	old := *pq
+	n := len(old)
+	item := old[n-1]
+	item.index = -1 // for safety
+	*pq = old[0 : n-1]
+	return item
+}
+
+//func (pq *containerPriorityQueue) update(item *containeritem, value *Bitmap, keyindex int) {
+//	item.value = value
+//	item.keyindex = keyindex
+//	heap.Fix(pq, item.index)
+//}

vendor/github.com/RoaringBitmap/roaring/roaringarray.go

@@ -0,0 +1,834 @@
+package roaring
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"io"
+
+	snappy "github.com/glycerine/go-unsnap-stream"
+	"github.com/tinylib/msgp/msgp"
+)
+
+//go:generate msgp -unexported
+
+type container interface {
+	addOffset(uint16) []container
+
+	clone() container
+	and(container) container
+	andCardinality(container) int
+	iand(container) container // i stands for inplace
+	andNot(container) container
+	iandNot(container) container // i stands for inplace
+	getCardinality() int
+	// rank returns the number of integers that are
+	// smaller or equal to x. rank(infinity) would be getCardinality().
+	rank(uint16) int
+
+	iadd(x uint16) bool                   // inplace, returns true if x was new.
+	iaddReturnMinimized(uint16) container // may change return type to minimize storage.
+
+	//addRange(start, final int) container  // range is [firstOfRange,lastOfRange) (unused)
+	iaddRange(start, endx int) container // i stands for inplace, range is [firstOfRange,endx)
+
+	iremove(x uint16) bool                   // inplace, returns true if x was present.
+	iremoveReturnMinimized(uint16) container // may change return type to minimize storage.
+
+	not(start, final int) container        // range is [firstOfRange,lastOfRange)
+	inot(firstOfRange, endx int) container // i stands for inplace, range is [firstOfRange,endx)
+	xor(r container) container
+	getShortIterator() shortPeekable
+	iterate(cb func(x uint16) bool) bool
+	getReverseIterator() shortIterable
+	getManyIterator() manyIterable
+	contains(i uint16) bool
+	maximum() uint16
+	minimum() uint16
+
+	// equals is now logical equals; it does not require the
+	// same underlying container types, but compares across
+	// any of the implementations.
+	equals(r container) bool
+
+	fillLeastSignificant16bits(array []uint32, i int, mask uint32)
+	or(r container) container
+	orCardinality(r container) int
+	isFull() bool
+	ior(r container) container   // i stands for inplace
+	intersects(r container) bool // whether the two containers intersect
+	lazyOR(r container) container
+	lazyIOR(r container) container
+	getSizeInBytes() int
+	//removeRange(start, final int) container  // range is [firstOfRange,lastOfRange) (unused)
+	iremoveRange(start, final int) container // i stands for inplace, range is [firstOfRange,lastOfRange)
+	selectInt(x uint16) int                  // selectInt returns the xth integer in the container
+	serializedSizeInBytes() int
+	writeTo(io.Writer) (int, error)
+
+	numberOfRuns() int
+	toEfficientContainer() container
+	String() string
+	containerType() contype
+}
+
+type contype uint8
+
+const (
+	bitmapContype contype = iota
+	arrayContype
+	run16Contype
+	run32Contype
+)
+
+// careful: range is [firstOfRange,lastOfRange]
+func rangeOfOnes(start, last int) container {
+	if start > MaxUint16 {
+		panic("rangeOfOnes called with start > MaxUint16")
+	}
+	if last > MaxUint16 {
+		panic("rangeOfOnes called with last > MaxUint16")
+	}
+	if start < 0 {
+		panic("rangeOfOnes called with start < 0")
+	}
+	if last < 0 {
+		panic("rangeOfOnes called with last < 0")
+	}
+	return newRunContainer16Range(uint16(start), uint16(last))
+}
+
+type roaringArray struct {
+	keys            []uint16
+	containers      []container `msg:"-"` // don't try to serialize directly.
+	needCopyOnWrite []bool
+	copyOnWrite     bool
+
+	// conserz is used at serialization time
+	// to serialize containers. Otherwise empty.
+	conserz []containerSerz
+}
+
+// containerSerz facilitates serializing container (tricky to
+// serialize because it is an interface) by providing a
+// light wrapper with a type identifier.
+type containerSerz struct {
+	t contype  `msg:"t"` // type
+	r msgp.Raw `msg:"r"` // Raw msgpack of the actual container type
+}
+
+func newRoaringArray() *roaringArray {
+	return &roaringArray{}
+}
+
+// runOptimize compresses the element containers to minimize space consumed.
+// Q: how does this interact with copyOnWrite and needCopyOnWrite?
+// A: since we aren't changing the logical content, just the representation,
+//    we don't bother to check the needCopyOnWrite bits. We replace
+//    (possibly all) elements of ra.containers in-place with space
+//    optimized versions.
+func (ra *roaringArray) runOptimize() {
+	for i := range ra.containers {
+		ra.containers[i] = ra.containers[i].toEfficientContainer()
+	}
+}
+
+func (ra *roaringArray) appendContainer(key uint16, value container, mustCopyOnWrite bool) {
+	ra.keys = append(ra.keys, key)
+	ra.containers = append(ra.containers, value)
+	ra.needCopyOnWrite = append(ra.needCopyOnWrite, mustCopyOnWrite)
+}
+
+func (ra *roaringArray) appendWithoutCopy(sa roaringArray, startingindex int) {
+	mustCopyOnWrite := sa.needCopyOnWrite[startingindex]
+	ra.appendContainer(sa.keys[startingindex], sa.containers[startingindex], mustCopyOnWrite)
+}
+
+func (ra *roaringArray) appendCopy(sa roaringArray, startingindex int) {
+	// cow only if the two request it, or if we already have a lightweight copy
+	copyonwrite := (ra.copyOnWrite && sa.copyOnWrite) || sa.needsCopyOnWrite(startingindex)
+	if !copyonwrite {
+		// since there is no copy-on-write, we need to clone the container (this is important)
+		ra.appendContainer(sa.keys[startingindex], sa.containers[startingindex].clone(), copyonwrite)
+	} else {
+		ra.appendContainer(sa.keys[startingindex], sa.containers[startingindex], copyonwrite)
+		if !sa.needsCopyOnWrite(startingindex) {
+			sa.setNeedsCopyOnWrite(startingindex)
+		}
+	}
+}
+
+func (ra *roaringArray) appendWithoutCopyMany(sa roaringArray, startingindex, end int) {
+	for i := startingindex; i < end; i++ {
+		ra.appendWithoutCopy(sa, i)
+	}
+}
+
+func (ra *roaringArray) appendCopyMany(sa roaringArray, startingindex, end int) {
+	for i := startingindex; i < end; i++ {
+		ra.appendCopy(sa, i)
+	}
+}
+
+func (ra *roaringArray) appendCopiesUntil(sa roaringArray, stoppingKey uint16) {
+	// cow only if the two request it, or if we already have a lightweight copy
+	copyonwrite := ra.copyOnWrite && sa.copyOnWrite
+
+	for i := 0; i < sa.size(); i++ {
+		if sa.keys[i] >= stoppingKey {
+			break
+		}
+		thiscopyonewrite := copyonwrite || sa.needsCopyOnWrite(i)
+		if thiscopyonewrite {
+			ra.appendContainer(sa.keys[i], sa.containers[i], thiscopyonewrite)
+			if !sa.needsCopyOnWrite(i) {
+				sa.setNeedsCopyOnWrite(i)
+			}
+
+		} else {
+			// since there is no copy-on-write, we need to clone the container (this is important)
+			ra.appendContainer(sa.keys[i], sa.containers[i].clone(), thiscopyonewrite)
+
+		}
+	}
+}
+
+func (ra *roaringArray) appendCopiesAfter(sa roaringArray, beforeStart uint16) {
+	// cow only if the two request it, or if we already have a lightweight copy
+	copyonwrite := ra.copyOnWrite && sa.copyOnWrite
+
+	startLocation := sa.getIndex(beforeStart)
+	if startLocation >= 0 {
+		startLocation++
+	} else {
+		startLocation = -startLocation - 1
+	}
+
+	for i := startLocation; i < sa.size(); i++ {
+		thiscopyonewrite := copyonwrite || sa.needsCopyOnWrite(i)
+		if thiscopyonewrite {
+			ra.appendContainer(sa.keys[i], sa.containers[i], thiscopyonewrite)
+			if !sa.needsCopyOnWrite(i) {
+				sa.setNeedsCopyOnWrite(i)
+			}
+		} else {
+			// since there is no copy-on-write, we need to clone the container (this is important)
+			ra.appendContainer(sa.keys[i], sa.containers[i].clone(), thiscopyonewrite)
+
+		}
+	}
+}
+
+func (ra *roaringArray) removeIndexRange(begin, end int) {
+	if end <= begin {
+		return
+	}
+
+	r := end - begin
+
+	copy(ra.keys[begin:], ra.keys[end:])
+	copy(ra.containers[begin:], ra.containers[end:])
+	copy(ra.needCopyOnWrite[begin:], ra.needCopyOnWrite[end:])
+
+	ra.resize(len(ra.keys) - r)
+}
+
+func (ra *roaringArray) resize(newsize int) {
+	for k := newsize; k < len(ra.containers); k++ {
+		ra.containers[k] = nil
+	}
+
+	ra.keys = ra.keys[:newsize]
+	ra.containers = ra.containers[:newsize]
+	ra.needCopyOnWrite = ra.needCopyOnWrite[:newsize]
+}
+
+func (ra *roaringArray) clear() {
+	ra.resize(0)
+	ra.copyOnWrite = false
+	ra.conserz = nil
+}
+
+func (ra *roaringArray) clone() *roaringArray {
+
+	sa := roaringArray{}
+	sa.copyOnWrite = ra.copyOnWrite
+
+	// this is where copyOnWrite is used.
+	if ra.copyOnWrite {
+		sa.keys = make([]uint16, len(ra.keys))
+		copy(sa.keys, ra.keys)
+		sa.containers = make([]container, len(ra.containers))
+		copy(sa.containers, ra.containers)
+		sa.needCopyOnWrite = make([]bool, len(ra.needCopyOnWrite))
+
+		ra.markAllAsNeedingCopyOnWrite()
+		sa.markAllAsNeedingCopyOnWrite()
+
+		// sa.needCopyOnWrite is shared
+	} else {
+		// make a full copy
+
+		sa.keys = make([]uint16, len(ra.keys))
+		copy(sa.keys, ra.keys)
+
+		sa.containers = make([]container, len(ra.containers))
+		for i := range sa.containers {
+			sa.containers[i] = ra.containers[i].clone()
+		}
+
+		sa.needCopyOnWrite = make([]bool, len(ra.needCopyOnWrite))
+	}
+	return &sa
+}
+
+// clone all containers which have needCopyOnWrite set to true
+// This can be used to make sure it is safe to munmap a []byte
+// that the roaring array may still have a reference to.
+func (ra *roaringArray) cloneCopyOnWriteContainers() {
+	for i, needCopyOnWrite := range ra.needCopyOnWrite {
+		if needCopyOnWrite {
+			ra.containers[i] = ra.containers[i].clone()
+			ra.needCopyOnWrite[i] = false
+		}
+	}
+}
+
+// unused function:
+//func (ra *roaringArray) containsKey(x uint16) bool {
+//	return (ra.binarySearch(0, int64(len(ra.keys)), x) >= 0)
+//}
+
+func (ra *roaringArray) getContainer(x uint16) container {
+	i := ra.binarySearch(0, int64(len(ra.keys)), x)
+	if i < 0 {
+		return nil
+	}
+	return ra.containers[i]
+}
+
+func (ra *roaringArray) getContainerAtIndex(i int) container {
+	return ra.containers[i]
+}
+
+func (ra *roaringArray) getFastContainerAtIndex(i int, needsWriteable bool) container {
+	c := ra.getContainerAtIndex(i)
+	switch t := c.(type) {
+	case *arrayContainer:
+		c = t.toBitmapContainer()
+	case *runContainer16:
+		if !t.isFull() {
+			c = t.toBitmapContainer()
+		}
+	case *bitmapContainer:
+		if needsWriteable && ra.needCopyOnWrite[i] {
+			c = ra.containers[i].clone()
+		}
+	}
+	return c
+}
+
+func (ra *roaringArray) getWritableContainerAtIndex(i int) container {
+	if ra.needCopyOnWrite[i] {
+		ra.containers[i] = ra.containers[i].clone()
+		ra.needCopyOnWrite[i] = false
+	}
+	return ra.containers[i]
+}
+
+func (ra *roaringArray) getIndex(x uint16) int {
+	// before the binary search, we optimize for frequent cases
+	size := len(ra.keys)
+	if (size == 0) || (ra.keys[size-1] == x) {
+		return size - 1
+	}
+	return ra.binarySearch(0, int64(size), x)
+}
+
+func (ra *roaringArray) getKeyAtIndex(i int) uint16 {
+	return ra.keys[i]
+}
+
+func (ra *roaringArray) insertNewKeyValueAt(i int, key uint16, value container) {
+	ra.keys = append(ra.keys, 0)
+	ra.containers = append(ra.containers, nil)
+
+	copy(ra.keys[i+1:], ra.keys[i:])
+	copy(ra.containers[i+1:], ra.containers[i:])
+
+	ra.keys[i] = key
+	ra.containers[i] = value
+
+	ra.needCopyOnWrite = append(ra.needCopyOnWrite, false)
+	copy(ra.needCopyOnWrite[i+1:], ra.needCopyOnWrite[i:])
+	ra.needCopyOnWrite[i] = false
+}
+
+func (ra *roaringArray) remove(key uint16) bool {
+	i := ra.binarySearch(0, int64(len(ra.keys)), key)
+	if i >= 0 { // if a new key
+		ra.removeAtIndex(i)
+		return true
+	}
+	return false
+}
+
+func (ra *roaringArray) removeAtIndex(i int) {
+	copy(ra.keys[i:], ra.keys[i+1:])
+	copy(ra.containers[i:], ra.containers[i+1:])
+
+	copy(ra.needCopyOnWrite[i:], ra.needCopyOnWrite[i+1:])
+
+	ra.resize(len(ra.keys) - 1)
+}
+
+func (ra *roaringArray) setContainerAtIndex(i int, c container) {
+	ra.containers[i] = c
+}
+
+func (ra *roaringArray) replaceKeyAndContainerAtIndex(i int, key uint16, c container, mustCopyOnWrite bool) {
+	ra.keys[i] = key
+	ra.containers[i] = c
+	ra.needCopyOnWrite[i] = mustCopyOnWrite
+}
+
+func (ra *roaringArray) size() int {
+	return len(ra.keys)
+}
+
+func (ra *roaringArray) binarySearch(begin, end int64, ikey uint16) int {
+	low := begin
+	high := end - 1
+	for low+16 <= high {
+		middleIndex := low + (high-low)/2 // avoid overflow
+		middleValue := ra.keys[middleIndex]
+
+		if middleValue < ikey {
+			low = middleIndex + 1
+		} else if middleValue > ikey {
+			high = middleIndex - 1
+		} else {
+			return int(middleIndex)
+		}
+	}
+	for ; low <= high; low++ {
+		val := ra.keys[low]
+		if val >= ikey {
+			if val == ikey {
+				return int(low)
+			}
+			break
+		}
+	}
+	return -int(low + 1)
+}
+
+func (ra *roaringArray) equals(o interface{}) bool {
+	srb, ok := o.(roaringArray)
+	if ok {
+
+		if srb.size() != ra.size() {
+			return false
+		}
+		for i, k := range ra.keys {
+			if k != srb.keys[i] {
+				return false
+			}
+		}
+
+		for i, c := range ra.containers {
+			if !c.equals(srb.containers[i]) {
+				return false
+			}
+		}
+		return true
+	}
+	return false
+}
+
+func (ra *roaringArray) headerSize() uint64 {
+	size := uint64(len(ra.keys))
+	if ra.hasRunCompression() {
+		if size < noOffsetThreshold { // for small bitmaps, we omit the offsets
+			return 4 + (size+7)/8 + 4*size
+		}
+		return 4 + (size+7)/8 + 8*size // - 4 because we pack the size with the cookie
+	}
+	return 4 + 4 + 8*size
+
+}
+
+// should be dirt cheap
+func (ra *roaringArray) serializedSizeInBytes() uint64 {
+	answer := ra.headerSize()
+	for _, c := range ra.containers {
+		answer += uint64(c.serializedSizeInBytes())
+	}
+	return answer
+}
+
+//
+// spec: https://github.com/RoaringBitmap/RoaringFormatSpec
+//
+func (ra *roaringArray) writeTo(w io.Writer) (n int64, err error) {
+	hasRun := ra.hasRunCompression()
+	isRunSizeInBytes := 0
+	cookieSize := 8
+	if hasRun {
+		cookieSize = 4
+		isRunSizeInBytes = (len(ra.keys) + 7) / 8
+	}
+	descriptiveHeaderSize := 4 * len(ra.keys)
+	preambleSize := cookieSize + isRunSizeInBytes + descriptiveHeaderSize
+
+	buf := make([]byte, preambleSize+4*len(ra.keys))
+
+	nw := 0
+
+	if hasRun {
+		binary.LittleEndian.PutUint16(buf[0:], uint16(serialCookie))
+		nw += 2
+		binary.LittleEndian.PutUint16(buf[2:], uint16(len(ra.keys)-1))
+		nw += 2
+		// compute isRun bitmap without temporary allocation
+		var runbitmapslice = buf[nw:nw+isRunSizeInBytes]
+		for i, c := range ra.containers {
+			switch c.(type) {
+			case *runContainer16:
+				runbitmapslice[i / 8] |= 1<<(uint(i)%8)
+			}
+		}
+		nw += isRunSizeInBytes
+	} else {
+		binary.LittleEndian.PutUint32(buf[0:], uint32(serialCookieNoRunContainer))
+		nw += 4
+		binary.LittleEndian.PutUint32(buf[4:], uint32(len(ra.keys)))
+		nw += 4
+	}
+
+	// descriptive header
+	for i, key := range ra.keys {
+		binary.LittleEndian.PutUint16(buf[nw:], key)
+		nw += 2
+		c := ra.containers[i]
+		binary.LittleEndian.PutUint16(buf[nw:], uint16(c.getCardinality()-1))
+		nw += 2
+	}
+
+	startOffset := int64(preambleSize + 4*len(ra.keys))
+	if !hasRun || (len(ra.keys) >= noOffsetThreshold) {
+		// offset header
+		for _, c := range ra.containers {
+			binary.LittleEndian.PutUint32(buf[nw:], uint32(startOffset))
+			nw += 4
+			switch rc := c.(type) {
+			case *runContainer16:
+				startOffset += 2 + int64(len(rc.iv))*4
+			default:
+				startOffset += int64(getSizeInBytesFromCardinality(c.getCardinality()))
+			}
+		}
+	}
+
+	written, err := w.Write(buf[:nw])
+	if err != nil {
+		return n, err
+	}
+	n += int64(written)
+
+	for _, c := range ra.containers {
+		written, err := c.writeTo(w)
+		if err != nil {
+			return n, err
+		}
+		n += int64(written)
+	}
+	return n, nil
+}
+
+//
+// spec: https://github.com/RoaringBitmap/RoaringFormatSpec
+//
+func (ra *roaringArray) toBytes() ([]byte, error) {
+	var buf bytes.Buffer
+	_, err := ra.writeTo(&buf)
+	return buf.Bytes(), err
+}
+
+func (ra *roaringArray) readFrom(stream byteInput) (int64, error) {
+	cookie, err := stream.readUInt32()
+
+	if err != nil {
+		return stream.getReadBytes(), fmt.Errorf("error in roaringArray.readFrom: could not read initial cookie: %s", err)
+	}
+
+	var size uint32
+	var isRunBitmap []byte
+
+	if cookie&0x0000FFFF == serialCookie {
+		size = uint32(uint16(cookie>>16) + 1)
+		// create is-run-container bitmap
+		isRunBitmapSize := (int(size) + 7) / 8
+		isRunBitmap, err = stream.next(isRunBitmapSize)
+
+		if err != nil {
+			return stream.getReadBytes(), fmt.Errorf("malformed bitmap, failed to read is-run bitmap, got: %s", err)
+		}
+	} else if cookie == serialCookieNoRunContainer {
+		size, err = stream.readUInt32()
+
+		if err != nil {
+			return stream.getReadBytes(), fmt.Errorf("malformed bitmap, failed to read a bitmap size: %s", err)
+		}
+	} else {
+		return stream.getReadBytes(), fmt.Errorf("error in roaringArray.readFrom: did not find expected serialCookie in header")
+	}
+
+	if size > (1 << 16) {
+		return stream.getReadBytes(), fmt.Errorf("it is logically impossible to have more than (1<<16) containers")
+	}
+
+	// descriptive header
+	buf, err := stream.next(2 * 2 * int(size))
+
+	if err != nil {
+		return stream.getReadBytes(), fmt.Errorf("failed to read descriptive header: %s", err)
+	}
+
+	keycard := byteSliceAsUint16Slice(buf)
+
+	if isRunBitmap == nil || size >= noOffsetThreshold {
+		if err := stream.skipBytes(int(size) * 4); err != nil {
+			return stream.getReadBytes(), fmt.Errorf("failed to skip bytes: %s", err)
+		}
+	}
+
+	// Allocate slices upfront as number of containers is known
+	if cap(ra.containers) >= int(size) {
+		ra.containers = ra.containers[:size]
+	} else {
+		ra.containers = make([]container, size)
+	}
+
+	if cap(ra.keys) >= int(size) {
+		ra.keys = ra.keys[:size]
+	} else {
+		ra.keys = make([]uint16, size)
+	}
+
+	if cap(ra.needCopyOnWrite) >= int(size) {
+		ra.needCopyOnWrite = ra.needCopyOnWrite[:size]
+	} else {
+		ra.needCopyOnWrite = make([]bool, size)
+	}
+
+	for i := uint32(0); i < size; i++ {
+		key := keycard[2*i]
+		card := int(keycard[2*i+1]) + 1
+		ra.keys[i] = key
+		ra.needCopyOnWrite[i] = true
+
+		if isRunBitmap != nil && isRunBitmap[i/8]&(1<<(i%8)) != 0 {
+			// run container
+			nr, err := stream.readUInt16()
+
+			if err != nil {
+				return 0, fmt.Errorf("failed to read runtime container size: %s", err)
+			}
+
+			buf, err := stream.next(int(nr) * 4)
+
+			if err != nil {
+				return stream.getReadBytes(), fmt.Errorf("failed to read runtime container content: %s", err)
+			}
+
+			nb := runContainer16{
+				iv:   byteSliceAsInterval16Slice(buf),
+				card: int64(card),
+			}
+
+			ra.containers[i] = &nb
+		} else if card > arrayDefaultMaxSize {
+			// bitmap container
+			buf, err := stream.next(arrayDefaultMaxSize * 2)
+
+			if err != nil {
+				return stream.getReadBytes(), fmt.Errorf("failed to read bitmap container: %s", err)
+			}
+
+			nb := bitmapContainer{
+				cardinality: card,
+				bitmap:      byteSliceAsUint64Slice(buf),
+			}
+
+			ra.containers[i] = &nb
+		} else {
+			// array container
+			buf, err := stream.next(card * 2)
+
+			if err != nil {
+				return stream.getReadBytes(), fmt.Errorf("failed to read array container: %s", err)
+			}
+
+			nb := arrayContainer{
+				byteSliceAsUint16Slice(buf),
+			}
+
+			ra.containers[i] = &nb
+		}
+	}
+
+	return stream.getReadBytes(), nil
+}
+
+func (ra *roaringArray) hasRunCompression() bool {
+	for _, c := range ra.containers {
+		switch c.(type) {
+		case *runContainer16:
+			return true
+		}
+	}
+	return false
+}
+
+func (ra *roaringArray) writeToMsgpack(stream io.Writer) error {
+
+	ra.conserz = make([]containerSerz, len(ra.containers))
+	for i, v := range ra.containers {
+		switch cn := v.(type) {
+		case *bitmapContainer:
+			bts, err := cn.MarshalMsg(nil)
+			if err != nil {
+				return err
+			}
+			ra.conserz[i].t = bitmapContype
+			ra.conserz[i].r = bts
+		case *arrayContainer:
+			bts, err := cn.MarshalMsg(nil)
+			if err != nil {
+				return err
+			}
+			ra.conserz[i].t = arrayContype
+			ra.conserz[i].r = bts
+		case *runContainer16:
+			bts, err := cn.MarshalMsg(nil)
+			if err != nil {
+				return err
+			}
+			ra.conserz[i].t = run16Contype
+			ra.conserz[i].r = bts
+		default:
+			panic(fmt.Errorf("Unrecognized container implementation: %T", cn))
+		}
+	}
+	w := snappy.NewWriter(stream)
+	err := msgp.Encode(w, ra)
+	ra.conserz = nil
+	return err
+}
+
+func (ra *roaringArray) readFromMsgpack(stream io.Reader) error {
+	r := snappy.NewReader(stream)
+	err := msgp.Decode(r, ra)
+	if err != nil {
+		return err
+	}
+
+	if len(ra.containers) != len(ra.keys) {
+		ra.containers = make([]container, len(ra.keys))
+	}
+
+	for i, v := range ra.conserz {
+		switch v.t {
+		case bitmapContype:
+			c := &bitmapContainer{}
+			_, err = c.UnmarshalMsg(v.r)
+			if err != nil {
+				return err
+			}
+			ra.containers[i] = c
+		case arrayContype:
+			c := &arrayContainer{}
+			_, err = c.UnmarshalMsg(v.r)
+			if err != nil {
+				return err
+			}
+			ra.containers[i] = c
+		case run16Contype:
+			c := &runContainer16{}
+			_, err = c.UnmarshalMsg(v.r)
+			if err != nil {
+				return err
+			}
+			ra.containers[i] = c
+		default:
+			return fmt.Errorf("unrecognized contype serialization code: '%v'", v.t)
+		}
+	}
+	ra.conserz = nil
+	return nil
+}
+
+func (ra *roaringArray) advanceUntil(min uint16, pos int) int {
+	lower := pos + 1
+
+	if lower >= len(ra.keys) || ra.keys[lower] >= min {
+		return lower
+	}
+
+	spansize := 1
+
+	for lower+spansize < len(ra.keys) && ra.keys[lower+spansize] < min {
+		spansize *= 2
+	}
+	var upper int
+	if lower+spansize < len(ra.keys) {
+		upper = lower + spansize
+	} else {
+		upper = len(ra.keys) - 1
+	}
+
+	if ra.keys[upper] == min {
+		return upper
+	}
+
+	if ra.keys[upper] < min {
+		// means
+		// array
+		// has no
+		// item
+		// >= min
+		// pos = array.length;
+		return len(ra.keys)
+	}
+
+	// we know that the next-smallest span was too small
+	lower += (spansize >> 1)
+
+	mid := 0
+	for lower+1 != upper {
+		mid = (lower + upper) >> 1
+		if ra.keys[mid] == min {
+			return mid
+		} else if ra.keys[mid] < min {
+			lower = mid
+		} else {
+			upper = mid
+		}
+	}
+	return upper
+}
+
+func (ra *roaringArray) markAllAsNeedingCopyOnWrite() {
+	for i := range ra.needCopyOnWrite {
+		ra.needCopyOnWrite[i] = true
+	}
+}
+
+func (ra *roaringArray) needsCopyOnWrite(i int) bool {
+	return ra.needCopyOnWrite[i]
+}
+
+func (ra *roaringArray) setNeedsCopyOnWrite(i int) {
+	ra.needCopyOnWrite[i] = true
+}

vendor/github.com/RoaringBitmap/roaring/roaringarray_gen.go

@@ -0,0 +1,529 @@
+package roaring
+
+// NOTE: THIS FILE WAS PRODUCED BY THE
+// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp)
+// DO NOT EDIT
+
+import (
+	"github.com/tinylib/msgp/msgp"
+)
+
+// Deprecated: DecodeMsg implements msgp.Decodable
+func (z *containerSerz) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zxvk uint32
+	zxvk, err = dc.ReadMapHeader()
+	if err != nil {
+		return
+	}
+	for zxvk > 0 {
+		zxvk--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "t":
+			{
+				var zbzg uint8
+				zbzg, err = dc.ReadUint8()
+				z.t = contype(zbzg)
+			}
+			if err != nil {
+				return
+			}
+		case "r":
+			err = z.r.DecodeMsg(dc)
+			if err != nil {
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				return
+			}
+		}
+	}
+	return
+}
+
+// Deprecated: EncodeMsg implements msgp.Encodable
+func (z *containerSerz) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 2
+	// write "t"
+	err = en.Append(0x82, 0xa1, 0x74)
+	if err != nil {
+		return err
+	}
+	err = en.WriteUint8(uint8(z.t))
+	if err != nil {
+		return
+	}
+	// write "r"
+	err = en.Append(0xa1, 0x72)
+	if err != nil {
+		return err
+	}
+	err = z.r.EncodeMsg(en)
+	if err != nil {
+		return
+	}
+	return
+}
+
+// Deprecated: MarshalMsg implements msgp.Marshaler
+func (z *containerSerz) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 2
+	// string "t"
+	o = append(o, 0x82, 0xa1, 0x74)
+	o = msgp.AppendUint8(o, uint8(z.t))
+	// string "r"
+	o = append(o, 0xa1, 0x72)
+	o, err = z.r.MarshalMsg(o)
+	if err != nil {
+		return
+	}
+	return
+}
+
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
+func (z *containerSerz) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zbai uint32
+	zbai, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		return
+	}
+	for zbai > 0 {
+		zbai--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "t":
+			{
+				var zcmr uint8
+				zcmr, bts, err = msgp.ReadUint8Bytes(bts)
+				z.t = contype(zcmr)
+			}
+			if err != nil {
+				return
+			}
+		case "r":
+			bts, err = z.r.UnmarshalMsg(bts)
+			if err != nil {
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *containerSerz) Msgsize() (s int) {
+	s = 1 + 2 + msgp.Uint8Size + 2 + z.r.Msgsize()
+	return
+}
+
+// Deprecated: DecodeMsg implements msgp.Decodable
+func (z *contype) DecodeMsg(dc *msgp.Reader) (err error) {
+	{
+		var zajw uint8
+		zajw, err = dc.ReadUint8()
+		(*z) = contype(zajw)
+	}
+	if err != nil {
+		return
+	}
+	return
+}
+
+// Deprecated: EncodeMsg implements msgp.Encodable
+func (z contype) EncodeMsg(en *msgp.Writer) (err error) {
+	err = en.WriteUint8(uint8(z))
+	if err != nil {
+		return
+	}
+	return
+}
+
+// Deprecated: MarshalMsg implements msgp.Marshaler
+func (z contype) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	o = msgp.AppendUint8(o, uint8(z))
+	return
+}
+
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
+func (z *contype) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	{
+		var zwht uint8
+		zwht, bts, err = msgp.ReadUint8Bytes(bts)
+		(*z) = contype(zwht)
+	}
+	if err != nil {
+		return
+	}
+	o = bts
+	return
+}
+
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z contype) Msgsize() (s int) {
+	s = msgp.Uint8Size
+	return
+}
+
+// Deprecated: DecodeMsg implements msgp.Decodable
+func (z *roaringArray) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zlqf uint32
+	zlqf, err = dc.ReadMapHeader()
+	if err != nil {
+		return
+	}
+	for zlqf > 0 {
+		zlqf--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "keys":
+			var zdaf uint32
+			zdaf, err = dc.ReadArrayHeader()
+			if err != nil {
+				return
+			}
+			if cap(z.keys) >= int(zdaf) {
+				z.keys = (z.keys)[:zdaf]
+			} else {
+				z.keys = make([]uint16, zdaf)
+			}
+			for zhct := range z.keys {
+				z.keys[zhct], err = dc.ReadUint16()
+				if err != nil {
+					return
+				}
+			}
+		case "needCopyOnWrite":
+			var zpks uint32
+			zpks, err = dc.ReadArrayHeader()
+			if err != nil {
+				return
+			}
+			if cap(z.needCopyOnWrite) >= int(zpks) {
+				z.needCopyOnWrite = (z.needCopyOnWrite)[:zpks]
+			} else {
+				z.needCopyOnWrite = make([]bool, zpks)
+			}
+			for zcua := range z.needCopyOnWrite {
+				z.needCopyOnWrite[zcua], err = dc.ReadBool()
+				if err != nil {
+					return
+				}
+			}
+		case "copyOnWrite":
+			z.copyOnWrite, err = dc.ReadBool()
+			if err != nil {
+				return
+			}
+		case "conserz":
+			var zjfb uint32
+			zjfb, err = dc.ReadArrayHeader()
+			if err != nil {
+				return
+			}
+			if cap(z.conserz) >= int(zjfb) {
+				z.conserz = (z.conserz)[:zjfb]
+			} else {
+				z.conserz = make([]containerSerz, zjfb)
+			}
+			for zxhx := range z.conserz {
+				var zcxo uint32
+				zcxo, err = dc.ReadMapHeader()
+				if err != nil {
+					return
+				}
+				for zcxo > 0 {
+					zcxo--
+					field, err = dc.ReadMapKeyPtr()
+					if err != nil {
+						return
+					}
+					switch msgp.UnsafeString(field) {
+					case "t":
+						{
+							var zeff uint8
+							zeff, err = dc.ReadUint8()
+							z.conserz[zxhx].t = contype(zeff)
+						}
+						if err != nil {
+							return
+						}
+					case "r":
+						err = z.conserz[zxhx].r.DecodeMsg(dc)
+						if err != nil {
+							return
+						}
+					default:
+						err = dc.Skip()
+						if err != nil {
+							return
+						}
+					}
+				}
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				return
+			}
+		}
+	}
+	return
+}
+
+// Deprecated: EncodeMsg implements msgp.Encodable
+func (z *roaringArray) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 4
+	// write "keys"
+	err = en.Append(0x84, 0xa4, 0x6b, 0x65, 0x79, 0x73)
+	if err != nil {
+		return err
+	}
+	err = en.WriteArrayHeader(uint32(len(z.keys)))
+	if err != nil {
+		return
+	}
+	for zhct := range z.keys {
+		err = en.WriteUint16(z.keys[zhct])
+		if err != nil {
+			return
+		}
+	}
+	// write "needCopyOnWrite"
+	err = en.Append(0xaf, 0x6e, 0x65, 0x65, 0x64, 0x43, 0x6f, 0x70, 0x79, 0x4f, 0x6e, 0x57, 0x72, 0x69, 0x74, 0x65)
+	if err != nil {
+		return err
+	}
+	err = en.WriteArrayHeader(uint32(len(z.needCopyOnWrite)))
+	if err != nil {
+		return
+	}
+	for zcua := range z.needCopyOnWrite {
+		err = en.WriteBool(z.needCopyOnWrite[zcua])
+		if err != nil {
+			return
+		}
+	}
+	// write "copyOnWrite"
+	err = en.Append(0xab, 0x63, 0x6f, 0x70, 0x79, 0x4f, 0x6e, 0x57, 0x72, 0x69, 0x74, 0x65)
+	if err != nil {
+		return err
+	}
+	err = en.WriteBool(z.copyOnWrite)
+	if err != nil {
+		return
+	}
+	// write "conserz"
+	err = en.Append(0xa7, 0x63, 0x6f, 0x6e, 0x73, 0x65, 0x72, 0x7a)
+	if err != nil {
+		return err
+	}
+	err = en.WriteArrayHeader(uint32(len(z.conserz)))
+	if err != nil {
+		return
+	}
+	for zxhx := range z.conserz {
+		// map header, size 2
+		// write "t"
+		err = en.Append(0x82, 0xa1, 0x74)
+		if err != nil {
+			return err
+		}
+		err = en.WriteUint8(uint8(z.conserz[zxhx].t))
+		if err != nil {
+			return
+		}
+		// write "r"
+		err = en.Append(0xa1, 0x72)
+		if err != nil {
+			return err
+		}
+		err = z.conserz[zxhx].r.EncodeMsg(en)
+		if err != nil {
+			return
+		}
+	}
+	return
+}
+
+// Deprecated: MarshalMsg implements msgp.Marshaler
+func (z *roaringArray) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 4
+	// string "keys"
+	o = append(o, 0x84, 0xa4, 0x6b, 0x65, 0x79, 0x73)
+	o = msgp.AppendArrayHeader(o, uint32(len(z.keys)))
+	for zhct := range z.keys {
+		o = msgp.AppendUint16(o, z.keys[zhct])
+	}
+	// string "needCopyOnWrite"
+	o = append(o, 0xaf, 0x6e, 0x65, 0x65, 0x64, 0x43, 0x6f, 0x70, 0x79, 0x4f, 0x6e, 0x57, 0x72, 0x69, 0x74, 0x65)
+	o = msgp.AppendArrayHeader(o, uint32(len(z.needCopyOnWrite)))
+	for zcua := range z.needCopyOnWrite {
+		o = msgp.AppendBool(o, z.needCopyOnWrite[zcua])
+	}
+	// string "copyOnWrite"
+	o = append(o, 0xab, 0x63, 0x6f, 0x70, 0x79, 0x4f, 0x6e, 0x57, 0x72, 0x69, 0x74, 0x65)
+	o = msgp.AppendBool(o, z.copyOnWrite)
+	// string "conserz"
+	o = append(o, 0xa7, 0x63, 0x6f, 0x6e, 0x73, 0x65, 0x72, 0x7a)
+	o = msgp.AppendArrayHeader(o, uint32(len(z.conserz)))
+	for zxhx := range z.conserz {
+		// map header, size 2
+		// string "t"
+		o = append(o, 0x82, 0xa1, 0x74)
+		o = msgp.AppendUint8(o, uint8(z.conserz[zxhx].t))
+		// string "r"
+		o = append(o, 0xa1, 0x72)
+		o, err = z.conserz[zxhx].r.MarshalMsg(o)
+		if err != nil {
+			return
+		}
+	}
+	return
+}
+
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
+func (z *roaringArray) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zrsw uint32
+	zrsw, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		return
+	}
+	for zrsw > 0 {
+		zrsw--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "keys":
+			var zxpk uint32
+			zxpk, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			if err != nil {
+				return
+			}
+			if cap(z.keys) >= int(zxpk) {
+				z.keys = (z.keys)[:zxpk]
+			} else {
+				z.keys = make([]uint16, zxpk)
+			}
+			for zhct := range z.keys {
+				z.keys[zhct], bts, err = msgp.ReadUint16Bytes(bts)
+				if err != nil {
+					return
+				}
+			}
+		case "needCopyOnWrite":
+			var zdnj uint32
+			zdnj, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			if err != nil {
+				return
+			}
+			if cap(z.needCopyOnWrite) >= int(zdnj) {
+				z.needCopyOnWrite = (z.needCopyOnWrite)[:zdnj]
+			} else {
+				z.needCopyOnWrite = make([]bool, zdnj)
+			}
+			for zcua := range z.needCopyOnWrite {
+				z.needCopyOnWrite[zcua], bts, err = msgp.ReadBoolBytes(bts)
+				if err != nil {
+					return
+				}
+			}
+		case "copyOnWrite":
+			z.copyOnWrite, bts, err = msgp.ReadBoolBytes(bts)
+			if err != nil {
+				return
+			}
+		case "conserz":
+			var zobc uint32
+			zobc, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			if err != nil {
+				return
+			}
+			if cap(z.conserz) >= int(zobc) {
+				z.conserz = (z.conserz)[:zobc]
+			} else {
+				z.conserz = make([]containerSerz, zobc)
+			}
+			for zxhx := range z.conserz {
+				var zsnv uint32
+				zsnv, bts, err = msgp.ReadMapHeaderBytes(bts)
+				if err != nil {
+					return
+				}
+				for zsnv > 0 {
+					zsnv--
+					field, bts, err = msgp.ReadMapKeyZC(bts)
+					if err != nil {
+						return
+					}
+					switch msgp.UnsafeString(field) {
+					case "t":
+						{
+							var zkgt uint8
+							zkgt, bts, err = msgp.ReadUint8Bytes(bts)
+							z.conserz[zxhx].t = contype(zkgt)
+						}
+						if err != nil {
+							return
+						}
+					case "r":
+						bts, err = z.conserz[zxhx].r.UnmarshalMsg(bts)
+						if err != nil {
+							return
+						}
+					default:
+						bts, err = msgp.Skip(bts)
+						if err != nil {
+							return
+						}
+					}
+				}
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *roaringArray) Msgsize() (s int) {
+	s = 1 + 5 + msgp.ArrayHeaderSize + (len(z.keys) * (msgp.Uint16Size)) + 16 + msgp.ArrayHeaderSize + (len(z.needCopyOnWrite) * (msgp.BoolSize)) + 12 + msgp.BoolSize + 8 + msgp.ArrayHeaderSize
+	for zxhx := range z.conserz {
+		s += 1 + 2 + msgp.Uint8Size + 2 + z.conserz[zxhx].r.Msgsize()
+	}
+	return
+}

vendor/github.com/RoaringBitmap/roaring/serialization.go

@@ -0,0 +1,34 @@
+package roaring
+
+import (
+	"encoding/binary"
+	"io"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+// writeTo for runContainer16 follows this
+// spec: https://github.com/RoaringBitmap/RoaringFormatSpec
+//
+func (b *runContainer16) writeTo(stream io.Writer) (int, error) {
+	buf := make([]byte, 2+4*len(b.iv))
+	binary.LittleEndian.PutUint16(buf[0:], uint16(len(b.iv)))
+	for i, v := range b.iv {
+		binary.LittleEndian.PutUint16(buf[2+i*4:], v.start)
+		binary.LittleEndian.PutUint16(buf[2+2+i*4:], v.length)
+	}
+	return stream.Write(buf)
+}
+
+func (b *runContainer16) writeToMsgpack(stream io.Writer) (int, error) {
+	bts, err := b.MarshalMsg(nil)
+	if err != nil {
+		return 0, err
+	}
+	return stream.Write(bts)
+}
+
+func (b *runContainer16) readFromMsgpack(stream io.Reader) (int, error) {
+	err := msgp.Decode(stream, b)
+	return 0, err
+}

vendor/github.com/RoaringBitmap/roaring/serialization_generic.go

@@ -0,0 +1,133 @@
+// +build !amd64,!386 appengine
+
+package roaring
+
+import (
+	"encoding/binary"
+	"errors"
+	"io"
+)
+
+func (b *arrayContainer) writeTo(stream io.Writer) (int, error) {
+	buf := make([]byte, 2*len(b.content))
+	for i, v := range b.content {
+		base := i * 2
+		buf[base] = byte(v)
+		buf[base+1] = byte(v >> 8)
+	}
+	return stream.Write(buf)
+}
+
+func (b *arrayContainer) readFrom(stream io.Reader) (int, error) {
+	err := binary.Read(stream, binary.LittleEndian, b.content)
+	if err != nil {
+		return 0, err
+	}
+	return 2 * len(b.content), nil
+}
+
+func (b *bitmapContainer) writeTo(stream io.Writer) (int, error) {
+	if b.cardinality <= arrayDefaultMaxSize {
+		return 0, errors.New("refusing to write bitmap container with cardinality of array container")
+	}
+
+	// Write set
+	buf := make([]byte, 8*len(b.bitmap))
+	for i, v := range b.bitmap {
+		base := i * 8
+		buf[base] = byte(v)
+		buf[base+1] = byte(v >> 8)
+		buf[base+2] = byte(v >> 16)
+		buf[base+3] = byte(v >> 24)
+		buf[base+4] = byte(v >> 32)
+		buf[base+5] = byte(v >> 40)
+		buf[base+6] = byte(v >> 48)
+		buf[base+7] = byte(v >> 56)
+	}
+	return stream.Write(buf)
+}
+
+func (b *bitmapContainer) readFrom(stream io.Reader) (int, error) {
+	err := binary.Read(stream, binary.LittleEndian, b.bitmap)
+	if err != nil {
+		return 0, err
+	}
+	b.computeCardinality()
+	return 8 * len(b.bitmap), nil
+}
+
+func (bc *bitmapContainer) asLittleEndianByteSlice() []byte {
+	by := make([]byte, len(bc.bitmap)*8)
+	for i := range bc.bitmap {
+		binary.LittleEndian.PutUint64(by[i*8:], bc.bitmap[i])
+	}
+	return by
+}
+
+func uint64SliceAsByteSlice(slice []uint64) []byte {
+	by := make([]byte, len(slice)*8)
+
+	for i, v := range slice {
+		binary.LittleEndian.PutUint64(by[i*8:], v)
+	}
+
+	return by
+}
+
+func uint16SliceAsByteSlice(slice []uint16) []byte {
+	by := make([]byte, len(slice)*2)
+
+	for i, v := range slice {
+		binary.LittleEndian.PutUint16(by[i*2:], v)
+	}
+
+	return by
+}
+
+func byteSliceAsUint16Slice(slice []byte) []uint16 {
+	if len(slice)%2 != 0 {
+		panic("Slice size should be divisible by 2")
+	}
+
+	b := make([]uint16, len(slice)/2)
+
+	for i := range b {
+		b[i] = binary.LittleEndian.Uint16(slice[2*i:])
+	}
+
+	return b
+}
+
+func byteSliceAsUint64Slice(slice []byte) []uint64 {
+	if len(slice)%8 != 0 {
+		panic("Slice size should be divisible by 8")
+	}
+
+	b := make([]uint64, len(slice)/8)
+
+	for i := range b {
+		b[i] = binary.LittleEndian.Uint64(slice[8*i:])
+	}
+
+	return b
+}
+
+// Converts a byte slice to a interval16 slice.
+// The function assumes that the slice byte buffer is run container data
+// encoded according to Roaring Format Spec
+func byteSliceAsInterval16Slice(byteSlice []byte) []interval16 {
+	if len(byteSlice)%4 != 0 {
+		panic("Slice size should be divisible by 4")
+	}
+
+	intervalSlice := make([]interval16, len(byteSlice)/4)
+
+	for i := range intervalSlice {
+		intervalSlice[i] = interval16{
+			start:  binary.LittleEndian.Uint16(byteSlice[i*4:]),
+			length: binary.LittleEndian.Uint16(byteSlice[i*4+2:]),
+		}
+	}
+
+	return intervalSlice
+}

vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go

@@ -0,0 +1,134 @@
+// +build 386 amd64,!appengine
+
+package roaring
+
+import (
+	"errors"
+	"io"
+	"reflect"
+	"runtime"
+	"unsafe"
+)
+
+func (ac *arrayContainer) writeTo(stream io.Writer) (int, error) {
+	buf := uint16SliceAsByteSlice(ac.content)
+	return stream.Write(buf)
+}
+
+func (bc *bitmapContainer) writeTo(stream io.Writer) (int, error) {
+	if bc.cardinality <= arrayDefaultMaxSize {
+		return 0, errors.New("refusing to write bitmap container with cardinality of array container")
+	}
+	buf := uint64SliceAsByteSlice(bc.bitmap)
+	return stream.Write(buf)
+}
+
+func uint64SliceAsByteSlice(slice []uint64) []byte {
+	// make a new slice header
+	header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice))
+
+	// update its capacity and length
+	header.Len *= 8
+	header.Cap *= 8
+
+	// instantiate result and use KeepAlive so data isn't unmapped.
+	result := *(*[]byte)(unsafe.Pointer(&header))
+	runtime.KeepAlive(&slice)
+
+	// return it
+	return result
+}
+
+func uint16SliceAsByteSlice(slice []uint16) []byte {
+	// make a new slice header
+	header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice))
+
+	// update its capacity and length
+	header.Len *= 2
+	header.Cap *= 2
+
+	// instantiate result and use KeepAlive so data isn't unmapped.
+	result := *(*[]byte)(unsafe.Pointer(&header))
+	runtime.KeepAlive(&slice)
+
+	// return it
+	return result
+}
+
+func (bc *bitmapContainer) asLittleEndianByteSlice() []byte {
+	return uint64SliceAsByteSlice(bc.bitmap)
+}
+
+// Deserialization code follows
+
+////
+// These methods (byteSliceAsUint16Slice,...) do not make copies,
+// they are pointer-based (unsafe). The caller is responsible to
+// ensure that the input slice does not get garbage collected, deleted
+// or modified while you hold the returned slince.
+////
+func byteSliceAsUint16Slice(slice []byte) (result []uint16) { // here we create a new slice holder
+	if len(slice)%2 != 0 {
+		panic("Slice size should be divisible by 2")
+	}
+	// reference: https://go101.org/article/unsafe.html
+
+	// make a new slice header
+	bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice))
+	rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result))
+
+	// transfer the data from the given slice to a new variable (our result)
+	rHeader.Data = bHeader.Data
+	rHeader.Len = bHeader.Len / 2
+	rHeader.Cap = bHeader.Cap / 2
+
+	// instantiate result and use KeepAlive so data isn't unmapped.
+	runtime.KeepAlive(&slice) // it is still crucial, GC can free it)
+
+	// return result
+	return
+}
+
+func byteSliceAsUint64Slice(slice []byte) (result []uint64) {
+	if len(slice)%8 != 0 {
+		panic("Slice size should be divisible by 8")
+	}
+	// reference: https://go101.org/article/unsafe.html
+
+	// make a new slice header
+	bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice))
+	rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result))
+
+	// transfer the data from the given slice to a new variable (our result)
+	rHeader.Data = bHeader.Data
+	rHeader.Len = bHeader.Len / 8
+	rHeader.Cap = bHeader.Cap / 8
+
+	// instantiate result and use KeepAlive so data isn't unmapped.
+	runtime.KeepAlive(&slice) // it is still crucial, GC can free it)
+
+	// return result
+	return
+}
+
+func byteSliceAsInterval16Slice(slice []byte) (result []interval16) {
+	if len(slice)%4 != 0 {
+		panic("Slice size should be divisible by 4")
+	}
+	// reference: https://go101.org/article/unsafe.html
+
+	// make a new slice header
+	bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice))
+	rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result))
+
+	// transfer the data from the given slice to a new variable (our result)
+	rHeader.Data = bHeader.Data
+	rHeader.Len = bHeader.Len / 4
+	rHeader.Cap = bHeader.Cap / 4
+
+	// instantiate result and use KeepAlive so data isn't unmapped.
+	runtime.KeepAlive(&slice) // it is still crucial, GC can free it)
+
+	// return result
+	return
+}

vendor/github.com/RoaringBitmap/roaring/serializationfuzz.go

@@ -0,0 +1,21 @@
+// +build gofuzz
+
+package roaring
+
+import "bytes"
+
+func FuzzSerializationStream(data []byte) int {
+	newrb := NewBitmap()
+	if _, err := newrb.ReadFrom(bytes.NewReader(data)); err != nil {
+		return 0
+	}
+	return 1
+}
+
+func FuzzSerializationBuffer(data []byte) int {
+	newrb := NewBitmap()
+	if _, err := newrb.FromBuffer(data); err != nil {
+		return 0
+	}
+	return 1
+}

vendor/github.com/RoaringBitmap/roaring/setutil.go

@@ -0,0 +1,610 @@
+package roaring
+
+func equal(a, b []uint16) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		if a[i] != b[i] {
+			return false
+		}
+	}
+	return true
+}
+
+func difference(set1 []uint16, set2 []uint16, buffer []uint16) int {
+	if 0 == len(set2) {
+		buffer = buffer[:len(set1)]
+		for k := 0; k < len(set1); k++ {
+			buffer[k] = set1[k]
+		}
+		return len(set1)
+	}
+	if 0 == len(set1) {
+		return 0
+	}
+	pos := 0
+	k1 := 0
+	k2 := 0
+	buffer = buffer[:cap(buffer)]
+	s1 := set1[k1]
+	s2 := set2[k2]
+	for {
+		if s1 < s2 {
+			buffer[pos] = s1
+			pos++
+			k1++
+			if k1 >= len(set1) {
+				break
+			}
+			s1 = set1[k1]
+		} else if s1 == s2 {
+			k1++
+			k2++
+			if k1 >= len(set1) {
+				break
+			}
+			s1 = set1[k1]
+			if k2 >= len(set2) {
+				for ; k1 < len(set1); k1++ {
+					buffer[pos] = set1[k1]
+					pos++
+				}
+				break
+			}
+			s2 = set2[k2]
+		} else { // if (val1>val2)
+			k2++
+			if k2 >= len(set2) {
+				for ; k1 < len(set1); k1++ {
+					buffer[pos] = set1[k1]
+					pos++
+				}
+				break
+			}
+			s2 = set2[k2]
+		}
+	}
+	return pos
+
+}
+
+func exclusiveUnion2by2(set1 []uint16, set2 []uint16, buffer []uint16) int {
+	if 0 == len(set2) {
+		buffer = buffer[:len(set1)]
+		copy(buffer, set1[:])
+		return len(set1)
+	}
+	if 0 == len(set1) {
+		buffer = buffer[:len(set2)]
+		copy(buffer, set2[:])
+		return len(set2)
+	}
+	pos := 0
+	k1 := 0
+	k2 := 0
+	s1 := set1[k1]
+	s2 := set2[k2]
+	buffer = buffer[:cap(buffer)]
+	for {
+		if s1 < s2 {
+			buffer[pos] = s1
+			pos++
+			k1++
+			if k1 >= len(set1) {
+				for ; k2 < len(set2); k2++ {
+					buffer[pos] = set2[k2]
+					pos++
+				}
+				break
+			}
+			s1 = set1[k1]
+		} else if s1 == s2 {
+			k1++
+			k2++
+			if k1 >= len(set1) {
+				for ; k2 < len(set2); k2++ {
+					buffer[pos] = set2[k2]
+					pos++
+				}
+				break
+			}
+			if k2 >= len(set2) {
+				for ; k1 < len(set1); k1++ {
+					buffer[pos] = set1[k1]
+					pos++
+				}
+				break
+			}
+			s1 = set1[k1]
+			s2 = set2[k2]
+		} else { // if (val1>val2)
+			buffer[pos] = s2
+			pos++
+			k2++
+			if k2 >= len(set2) {
+				for ; k1 < len(set1); k1++ {
+					buffer[pos] = set1[k1]
+					pos++
+				}
+				break
+			}
+			s2 = set2[k2]
+		}
+	}
+	return pos
+}
+
+func union2by2(set1 []uint16, set2 []uint16, buffer []uint16) int {
+	pos := 0
+	k1 := 0
+	k2 := 0
+	if 0 == len(set2) {
+		buffer = buffer[:len(set1)]
+		copy(buffer, set1[:])
+		return len(set1)
+	}
+	if 0 == len(set1) {
+		buffer = buffer[:len(set2)]
+		copy(buffer, set2[:])
+		return len(set2)
+	}
+	s1 := set1[k1]
+	s2 := set2[k2]
+	buffer = buffer[:cap(buffer)]
+	for {
+		if s1 < s2 {
+			buffer[pos] = s1
+			pos++
+			k1++
+			if k1 >= len(set1) {
+				copy(buffer[pos:], set2[k2:])
+				pos += len(set2) - k2
+				break
+			}
+			s1 = set1[k1]
+		} else if s1 == s2 {
+			buffer[pos] = s1
+			pos++
+			k1++
+			k2++
+			if k1 >= len(set1) {
+				copy(buffer[pos:], set2[k2:])
+				pos += len(set2) - k2
+				break
+			}
+			if k2 >= len(set2) {
+				copy(buffer[pos:], set1[k1:])
+				pos += len(set1) - k1
+				break
+			}
+			s1 = set1[k1]
+			s2 = set2[k2]
+		} else { // if (set1[k1]>set2[k2])
+			buffer[pos] = s2
+			pos++
+			k2++
+			if k2 >= len(set2) {
+				copy(buffer[pos:], set1[k1:])
+				pos += len(set1) - k1
+				break
+			}
+			s2 = set2[k2]
+		}
+	}
+	return pos
+}
+
+func union2by2Cardinality(set1 []uint16, set2 []uint16) int {
+	pos := 0
+	k1 := 0
+	k2 := 0
+	if 0 == len(set2) {
+		return len(set1)
+	}
+	if 0 == len(set1) {
+		return len(set2)
+	}
+	s1 := set1[k1]
+	s2 := set2[k2]
+	for {
+		if s1 < s2 {
+			pos++
+			k1++
+			if k1 >= len(set1) {
+				pos += len(set2) - k2
+				break
+			}
+			s1 = set1[k1]
+		} else if s1 == s2 {
+			pos++
+			k1++
+			k2++
+			if k1 >= len(set1) {
+				pos += len(set2) - k2
+				break
+			}
+			if k2 >= len(set2) {
+				pos += len(set1) - k1
+				break
+			}
+			s1 = set1[k1]
+			s2 = set2[k2]
+		} else { // if (set1[k1]>set2[k2])
+			pos++
+			k2++
+			if k2 >= len(set2) {
+				pos += len(set1) - k1
+				break
+			}
+			s2 = set2[k2]
+		}
+	}
+	return pos
+}
+
+func intersection2by2(
+	set1 []uint16,
+	set2 []uint16,
+	buffer []uint16) int {
+
+	if len(set1)*64 < len(set2) {
+		return onesidedgallopingintersect2by2(set1, set2, buffer)
+	} else if len(set2)*64 < len(set1) {
+		return onesidedgallopingintersect2by2(set2, set1, buffer)
+	} else {
+		return localintersect2by2(set1, set2, buffer)
+	}
+}
+
+func intersection2by2Cardinality(
+	set1 []uint16,
+	set2 []uint16) int {
+
+	if len(set1)*64 < len(set2) {
+		return onesidedgallopingintersect2by2Cardinality(set1, set2)
+	} else if len(set2)*64 < len(set1) {
+		return onesidedgallopingintersect2by2Cardinality(set2, set1)
+	} else {
+		return localintersect2by2Cardinality(set1, set2)
+	}
+}
+
+func intersects2by2(
+	set1 []uint16,
+	set2 []uint16) bool {
+	// could be optimized if one set is much larger than the other one
+	if (0 == len(set1)) || (0 == len(set2)) {
+		return false
+	}
+	k1 := 0
+	k2 := 0
+	s1 := set1[k1]
+	s2 := set2[k2]
+mainwhile:
+	for {
+
+		if s2 < s1 {
+			for {
+				k2++
+				if k2 == len(set2) {
+					break mainwhile
+				}
+				s2 = set2[k2]
+				if s2 >= s1 {
+					break
+				}
+			}
+		}
+		if s1 < s2 {
+			for {
+				k1++
+				if k1 == len(set1) {
+					break mainwhile
+				}
+				s1 = set1[k1]
+				if s1 >= s2 {
+					break
+				}
+			}
+
+		} else {
+			// (set2[k2] == set1[k1])
+			return true
+		}
+	}
+	return false
+}
+
+func localintersect2by2(
+	set1 []uint16,
+	set2 []uint16,
+	buffer []uint16) int {
+
+	if (0 == len(set1)) || (0 == len(set2)) {
+		return 0
+	}
+	k1 := 0
+	k2 := 0
+	pos := 0
+	buffer = buffer[:cap(buffer)]
+	s1 := set1[k1]
+	s2 := set2[k2]
+mainwhile:
+	for {
+		if s2 < s1 {
+			for {
+				k2++
+				if k2 == len(set2) {
+					break mainwhile
+				}
+				s2 = set2[k2]
+				if s2 >= s1 {
+					break
+				}
+			}
+		}
+		if s1 < s2 {
+			for {
+				k1++
+				if k1 == len(set1) {
+					break mainwhile
+				}
+				s1 = set1[k1]
+				if s1 >= s2 {
+					break
+				}
+			}
+
+		} else {
+			// (set2[k2] == set1[k1])
+			buffer[pos] = s1
+			pos++
+			k1++
+			if k1 == len(set1) {
+				break
+			}
+			s1 = set1[k1]
+			k2++
+			if k2 == len(set2) {
+				break
+			}
+			s2 = set2[k2]
+		}
+	}
+	return pos
+}
+
+func localintersect2by2Cardinality(
+	set1 []uint16,
+	set2 []uint16) int {
+
+	if (0 == len(set1)) || (0 == len(set2)) {
+		return 0
+	}
+	k1 := 0
+	k2 := 0
+	pos := 0
+	s1 := set1[k1]
+	s2 := set2[k2]
+mainwhile:
+	for {
+		if s2 < s1 {
+			for {
+				k2++
+				if k2 == len(set2) {
+					break mainwhile
+				}
+				s2 = set2[k2]
+				if s2 >= s1 {
+					break
+				}
+			}
+		}
+		if s1 < s2 {
+			for {
+				k1++
+				if k1 == len(set1) {
+					break mainwhile
+				}
+				s1 = set1[k1]
+				if s1 >= s2 {
+					break
+				}
+			}
+
+		} else {
+			// (set2[k2] == set1[k1])
+			pos++
+			k1++
+			if k1 == len(set1) {
+				break
+			}
+			s1 = set1[k1]
+			k2++
+			if k2 == len(set2) {
+				break
+			}
+			s2 = set2[k2]
+		}
+	}
+	return pos
+}
+
+func advanceUntil(
+	array []uint16,
+	pos int,
+	length int,
+	min uint16) int {
+	lower := pos + 1
+
+	if lower >= length || array[lower] >= min {
+		return lower
+	}
+
+	spansize := 1
+
+	for lower+spansize < length && array[lower+spansize] < min {
+		spansize *= 2
+	}
+	var upper int
+	if lower+spansize < length {
+		upper = lower + spansize
+	} else {
+		upper = length - 1
+	}
+
+	if array[upper] == min {
+		return upper
+	}
+
+	if array[upper] < min {
+		// means
+		// array
+		// has no
+		// item
+		// >= min
+		// pos = array.length;
+		return length
+	}
+
+	// we know that the next-smallest span was too small
+	lower += (spansize >> 1)
+
+	mid := 0
+	for lower+1 != upper {
+		mid = (lower + upper) >> 1
+		if array[mid] == min {
+			return mid
+		} else if array[mid] < min {
+			lower = mid
+		} else {
+			upper = mid
+		}
+	}
+	return upper
+
+}
+
+func onesidedgallopingintersect2by2(
+	smallset []uint16,
+	largeset []uint16,
+	buffer []uint16) int {
+
+	if 0 == len(smallset) {
+		return 0
+	}
+	buffer = buffer[:cap(buffer)]
+	k1 := 0
+	k2 := 0
+	pos := 0
+	s1 := largeset[k1]
+	s2 := smallset[k2]
+mainwhile:
+
+	for {
+		if s1 < s2 {
+			k1 = advanceUntil(largeset, k1, len(largeset), s2)
+			if k1 == len(largeset) {
+				break mainwhile
+			}
+			s1 = largeset[k1]
+		}
+		if s2 < s1 {
+			k2++
+			if k2 == len(smallset) {
+				break mainwhile
+			}
+			s2 = smallset[k2]
+		} else {
+
+			buffer[pos] = s2
+			pos++
+			k2++
+			if k2 == len(smallset) {
+				break
+			}
+			s2 = smallset[k2]
+			k1 = advanceUntil(largeset, k1, len(largeset), s2)
+			if k1 == len(largeset) {
+				break mainwhile
+			}
+			s1 = largeset[k1]
+		}
+
+	}
+	return pos
+}
+
+func onesidedgallopingintersect2by2Cardinality(
+	smallset []uint16,
+	largeset []uint16) int {
+
+	if 0 == len(smallset) {
+		return 0
+	}
+	k1 := 0
+	k2 := 0
+	pos := 0
+	s1 := largeset[k1]
+	s2 := smallset[k2]
+mainwhile:
+
+	for {
+		if s1 < s2 {
+			k1 = advanceUntil(largeset, k1, len(largeset), s2)
+			if k1 == len(largeset) {
+				break mainwhile
+			}
+			s1 = largeset[k1]
+		}
+		if s2 < s1 {
+			k2++
+			if k2 == len(smallset) {
+				break mainwhile
+			}
+			s2 = smallset[k2]
+		} else {
+
+			pos++
+			k2++
+			if k2 == len(smallset) {
+				break
+			}
+			s2 = smallset[k2]
+			k1 = advanceUntil(largeset, k1, len(largeset), s2)
+			if k1 == len(largeset) {
+				break mainwhile
+			}
+			s1 = largeset[k1]
+		}
+
+	}
+	return pos
+}
+
+func binarySearch(array []uint16, ikey uint16) int {
+	low := 0
+	high := len(array) - 1
+	for low+16 <= high {
+		middleIndex := int(uint32(low+high) >> 1)
+		middleValue := array[middleIndex]
+		if middleValue < ikey {
+			low = middleIndex + 1
+		} else if middleValue > ikey {
+			high = middleIndex - 1
+		} else {
+			return middleIndex
+		}
+	}
+	for ; low <= high; low++ {
+		val := array[low]
+		if val >= ikey {
+			if val == ikey {
+				return low
+			}
+			break
+		}
+	}
+	return -(low + 1)
+}

vendor/github.com/RoaringBitmap/roaring/shortiterator.go

@@ -0,0 +1,52 @@
+package roaring
+
+type shortIterable interface {
+	hasNext() bool
+	next() uint16
+}
+
+type shortPeekable interface {
+	shortIterable
+	peekNext() uint16
+	advanceIfNeeded(minval uint16)
+}
+
+type shortIterator struct {
+	slice []uint16
+	loc   int
+}
+
+func (si *shortIterator) hasNext() bool {
+	return si.loc < len(si.slice)
+}
+
+func (si *shortIterator) next() uint16 {
+	a := si.slice[si.loc]
+	si.loc++
+	return a
+}
+
+func (si *shortIterator) peekNext() uint16 {
+	return si.slice[si.loc]
+}
+
+func (si *shortIterator) advanceIfNeeded(minval uint16) {
+	if si.hasNext() && si.peekNext() < minval {
+		si.loc = advanceUntil(si.slice, si.loc, len(si.slice), minval)
+	}
+}
+
+type reverseIterator struct {
+	slice []uint16
+	loc   int
+}
+
+func (si *reverseIterator) hasNext() bool {
+	return si.loc >= 0
+}
+
+func (si *reverseIterator) next() uint16 {
+	a := si.slice[si.loc]
+	si.loc--
+	return a
+}

vendor/github.com/RoaringBitmap/roaring/smat.go

@@ -0,0 +1,383 @@
+// +build gofuzz
+
+/*
+# Instructions for smat testing for roaring
+
+[smat](https://github.com/mschoch/smat) is a framework that provides
+state machine assisted fuzz testing.
+
+To run the smat tests for roaring...
+
+## Prerequisites
+
+    $ go get github.com/dvyukov/go-fuzz/go-fuzz
+    $ go get github.com/dvyukov/go-fuzz/go-fuzz-build
+
+## Steps
+
+1.  Generate initial smat corpus:
+```
+    go test -tags=gofuzz -run=TestGenerateSmatCorpus
+```
+
+2.  Build go-fuzz test program with instrumentation:
+```
+    go-fuzz-build -func FuzzSmat github.com/RoaringBitmap/roaring
+```
+
+3.  Run go-fuzz:
+```
+    go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200
+```
+
+You should see output like...
+```
+2016/09/16 13:58:35 slaves: 8, corpus: 1 (3s ago), crashers: 0, restarts: 1/0, execs: 0 (0/sec), cover: 0, uptime: 3s
+2016/09/16 13:58:38 slaves: 8, corpus: 1 (6s ago), crashers: 0, restarts: 1/0, execs: 0 (0/sec), cover: 0, uptime: 6s
+2016/09/16 13:58:41 slaves: 8, corpus: 1 (9s ago), crashers: 0, restarts: 1/44, execs: 44 (5/sec), cover: 0, uptime: 9s
+2016/09/16 13:58:44 slaves: 8, corpus: 1 (12s ago), crashers: 0, restarts: 1/45, execs: 45 (4/sec), cover: 0, uptime: 12s
+2016/09/16 13:58:47 slaves: 8, corpus: 1 (15s ago), crashers: 0, restarts: 1/46, execs: 46 (3/sec), cover: 0, uptime: 15s
+2016/09/16 13:58:50 slaves: 8, corpus: 1 (18s ago), crashers: 0, restarts: 1/47, execs: 47 (3/sec), cover: 0, uptime: 18s
+2016/09/16 13:58:53 slaves: 8, corpus: 1 (21s ago), crashers: 0, restarts: 1/63, execs: 63 (3/sec), cover: 0, uptime: 21s
+2016/09/16 13:58:56 slaves: 8, corpus: 1 (24s ago), crashers: 0, restarts: 1/65, execs: 65 (3/sec), cover: 0, uptime: 24s
+2016/09/16 13:58:59 slaves: 8, corpus: 1 (27s ago), crashers: 0, restarts: 1/66, execs: 66 (2/sec), cover: 0, uptime: 27s
+2016/09/16 13:59:02 slaves: 8, corpus: 1 (30s ago), crashers: 0, restarts: 1/67, execs: 67 (2/sec), cover: 0, uptime: 30s
+2016/09/16 13:59:05 slaves: 8, corpus: 1 (33s ago), crashers: 0, restarts: 1/83, execs: 83 (3/sec), cover: 0, uptime: 33s
+2016/09/16 13:59:08 slaves: 8, corpus: 1 (36s ago), crashers: 0, restarts: 1/84, execs: 84 (2/sec), cover: 0, uptime: 36s
+2016/09/16 13:59:11 slaves: 8, corpus: 2 (0s ago), crashers: 0, restarts: 1/85, execs: 85 (2/sec), cover: 0, uptime: 39s
+2016/09/16 13:59:14 slaves: 8, corpus: 17 (2s ago), crashers: 0, restarts: 1/86, execs: 86 (2/sec), cover: 480, uptime: 42s
+2016/09/16 13:59:17 slaves: 8, corpus: 17 (5s ago), crashers: 0, restarts: 1/66, execs: 132 (3/sec), cover: 487, uptime: 45s
+2016/09/16 13:59:20 slaves: 8, corpus: 17 (8s ago), crashers: 0, restarts: 1/440, execs: 2645 (55/sec), cover: 487, uptime: 48s
+
+```
+
+Let it run, and if the # of crashers is > 0, check out the reports in
+the workdir where you should be able to find the panic goroutine stack
+traces.
+*/
+
+package roaring
+
+import (
+	"fmt"
+	"sort"
+
+	"github.com/mschoch/smat"
+	"github.com/willf/bitset"
+)
+
+// fuzz test using state machine driven by byte stream.
+func FuzzSmat(data []byte) int {
+	return smat.Fuzz(&smatContext{}, smat.ActionID('S'), smat.ActionID('T'),
+		smatActionMap, data)
+}
+
+var smatDebug = false
+
+func smatLog(prefix, format string, args ...interface{}) {
+	if smatDebug {
+		fmt.Print(prefix)
+		fmt.Printf(format, args...)
+	}
+}
+
+type smatContext struct {
+	pairs []*smatPair
+
+	// Two registers, x & y.
+	x int
+	y int
+
+	actions int
+}
+
+type smatPair struct {
+	bm *Bitmap
+	bs *bitset.BitSet
+}
+
+// ------------------------------------------------------------------
+
+var smatActionMap = smat.ActionMap{
+	smat.ActionID('X'): smatAction("x++", smatWrap(func(c *smatContext) { c.x++ })),
+	smat.ActionID('x'): smatAction("x--", smatWrap(func(c *smatContext) { c.x-- })),
+	smat.ActionID('Y'): smatAction("y++", smatWrap(func(c *smatContext) { c.y++ })),
+	smat.ActionID('y'): smatAction("y--", smatWrap(func(c *smatContext) { c.y-- })),
+	smat.ActionID('*'): smatAction("x*y", smatWrap(func(c *smatContext) { c.x = c.x * c.y })),
+	smat.ActionID('<'): smatAction("x<<", smatWrap(func(c *smatContext) { c.x = c.x << 1 })),
+
+	smat.ActionID('^'): smatAction("swap", smatWrap(func(c *smatContext) { c.x, c.y = c.y, c.x })),
+
+	smat.ActionID('['): smatAction(" pushPair", smatWrap(smatPushPair)),
+	smat.ActionID(']'): smatAction(" popPair", smatWrap(smatPopPair)),
+
+	smat.ActionID('B'): smatAction(" setBit", smatWrap(smatSetBit)),
+	smat.ActionID('b'): smatAction(" removeBit", smatWrap(smatRemoveBit)),
+
+	smat.ActionID('o'): smatAction(" or", smatWrap(smatOr)),
+	smat.ActionID('a'): smatAction(" and", smatWrap(smatAnd)),
+
+	smat.ActionID('#'): smatAction(" cardinality", smatWrap(smatCardinality)),
+
+	smat.ActionID('O'): smatAction(" orCardinality", smatWrap(smatOrCardinality)),
+	smat.ActionID('A'): smatAction(" andCardinality", smatWrap(smatAndCardinality)),
+
+	smat.ActionID('c'): smatAction(" clear", smatWrap(smatClear)),
+	smat.ActionID('r'): smatAction(" runOptimize", smatWrap(smatRunOptimize)),
+
+	smat.ActionID('e'): smatAction(" isEmpty", smatWrap(smatIsEmpty)),
+
+	smat.ActionID('i'): smatAction(" intersects", smatWrap(smatIntersects)),
+
+	smat.ActionID('f'): smatAction(" flip", smatWrap(smatFlip)),
+
+	smat.ActionID('-'): smatAction(" difference", smatWrap(smatDifference)),
+}
+
+var smatRunningPercentActions []smat.PercentAction
+
+func init() {
+	var ids []int
+	for actionId := range smatActionMap {
+		ids = append(ids, int(actionId))
+	}
+	sort.Ints(ids)
+
+	pct := 100 / len(smatActionMap)
+	for _, actionId := range ids {
+		smatRunningPercentActions = append(smatRunningPercentActions,
+			smat.PercentAction{pct, smat.ActionID(actionId)})
+	}
+
+	smatActionMap[smat.ActionID('S')] = smatAction("SETUP", smatSetupFunc)
+	smatActionMap[smat.ActionID('T')] = smatAction("TEARDOWN", smatTeardownFunc)
+}
+
+// We only have one smat state: running.
+func smatRunning(next byte) smat.ActionID {
+	return smat.PercentExecute(next, smatRunningPercentActions...)
+}
+
+func smatAction(name string, f func(ctx smat.Context) (smat.State, error)) func(smat.Context) (smat.State, error) {
+	return func(ctx smat.Context) (smat.State, error) {
+		c := ctx.(*smatContext)
+		c.actions++
+
+		smatLog("  ", "%s\n", name)
+
+		return f(ctx)
+	}
+}
+
+// Creates an smat action func based on a simple callback.
+func smatWrap(cb func(c *smatContext)) func(smat.Context) (next smat.State, err error) {
+	return func(ctx smat.Context) (next smat.State, err error) {
+		c := ctx.(*smatContext)
+		cb(c)
+		return smatRunning, nil
+	}
+}
+
+// Invokes a callback function with the input v bounded to len(c.pairs).
+func (c *smatContext) withPair(v int, cb func(*smatPair)) {
+	if len(c.pairs) > 0 {
+		if v < 0 {
+			v = -v
+		}
+		v = v % len(c.pairs)
+		cb(c.pairs[v])
+	}
+}
+
+// ------------------------------------------------------------------
+
+func smatSetupFunc(ctx smat.Context) (next smat.State, err error) {
+	return smatRunning, nil
+}
+
+func smatTeardownFunc(ctx smat.Context) (next smat.State, err error) {
+	return nil, err
+}
+
+// ------------------------------------------------------------------
+
+func smatPushPair(c *smatContext) {
+	c.pairs = append(c.pairs, &smatPair{
+		bm: NewBitmap(),
+		bs: bitset.New(100),
+	})
+}
+
+func smatPopPair(c *smatContext) {
+	if len(c.pairs) > 0 {
+		c.pairs = c.pairs[0 : len(c.pairs)-1]
+	}
+}
+
+func smatSetBit(c *smatContext) {
+	c.withPair(c.x, func(p *smatPair) {
+		y := uint32(c.y)
+		p.bm.AddInt(int(y))
+		p.bs.Set(uint(y))
+		p.checkEquals()
+	})
+}
+
+func smatRemoveBit(c *smatContext) {
+	c.withPair(c.x, func(p *smatPair) {
+		y := uint32(c.y)
+		p.bm.Remove(y)
+		p.bs.Clear(uint(y))
+		p.checkEquals()
+	})
+}
+
+func smatAnd(c *smatContext) {
+	c.withPair(c.x, func(px *smatPair) {
+		c.withPair(c.y, func(py *smatPair) {
+			px.bm.And(py.bm)
+			px.bs = px.bs.Intersection(py.bs)
+			px.checkEquals()
+			py.checkEquals()
+		})
+	})
+}
+
+func smatOr(c *smatContext) {
+	c.withPair(c.x, func(px *smatPair) {
+		c.withPair(c.y, func(py *smatPair) {
+			px.bm.Or(py.bm)
+			px.bs = px.bs.Union(py.bs)
+			px.checkEquals()
+			py.checkEquals()
+		})
+	})
+}
+
+func smatAndCardinality(c *smatContext) {
+	c.withPair(c.x, func(px *smatPair) {
+		c.withPair(c.y, func(py *smatPair) {
+			c0 := px.bm.AndCardinality(py.bm)
+			c1 := px.bs.IntersectionCardinality(py.bs)
+			if c0 != uint64(c1) {
+				panic("expected same add cardinality")
+			}
+			px.checkEquals()
+			py.checkEquals()
+		})
+	})
+}
+
+func smatOrCardinality(c *smatContext) {
+	c.withPair(c.x, func(px *smatPair) {
+		c.withPair(c.y, func(py *smatPair) {
+			c0 := px.bm.OrCardinality(py.bm)
+			c1 := px.bs.UnionCardinality(py.bs)
+			if c0 != uint64(c1) {
+				panic("expected same or cardinality")
+			}
+			px.checkEquals()
+			py.checkEquals()
+		})
+	})
+}
+
+func smatRunOptimize(c *smatContext) {
+	c.withPair(c.x, func(px *smatPair) {
+		px.bm.RunOptimize()
+		px.checkEquals()
+	})
+}
+
+func smatClear(c *smatContext) {
+	c.withPair(c.x, func(px *smatPair) {
+		px.bm.Clear()
+		px.bs = px.bs.ClearAll()
+		px.checkEquals()
+	})
+}
+
+func smatCardinality(c *smatContext) {
+	c.withPair(c.x, func(px *smatPair) {
+		c0 := px.bm.GetCardinality()
+		c1 := px.bs.Count()
+		if c0 != uint64(c1) {
+			panic("expected same cardinality")
+		}
+	})
+}
+
+func smatIsEmpty(c *smatContext) {
+	c.withPair(c.x, func(px *smatPair) {
+		c0 := px.bm.IsEmpty()
+		c1 := px.bs.None()
+		if c0 != c1 {
+			panic("expected same is empty")
+		}
+	})
+}
+
+func smatIntersects(c *smatContext) {
+	c.withPair(c.x, func(px *smatPair) {
+		c.withPair(c.y, func(py *smatPair) {
+			v0 := px.bm.Intersects(py.bm)
+			v1 := px.bs.IntersectionCardinality(py.bs) > 0
+			if v0 != v1 {
+				panic("intersects not equal")
+			}
+
+			px.checkEquals()
+			py.checkEquals()
+		})
+	})
+}
+
+func smatFlip(c *smatContext) {
+	c.withPair(c.x, func(p *smatPair) {
+		y := uint32(c.y)
+		p.bm.Flip(uint64(y), uint64(y)+1)
+		p.bs = p.bs.Flip(uint(y))
+		p.checkEquals()
+	})
+}
+
+func smatDifference(c *smatContext) {
+	c.withPair(c.x, func(px *smatPair) {
+		c.withPair(c.y, func(py *smatPair) {
+			px.bm.AndNot(py.bm)
+			px.bs = px.bs.Difference(py.bs)
+			px.checkEquals()
+			py.checkEquals()
+		})
+	})
+}
+
+func (p *smatPair) checkEquals() {
+	if !p.equalsBitSet(p.bs, p.bm) {
+		panic("bitset mismatch")
+	}
+}
+
+func (p *smatPair) equalsBitSet(a *bitset.BitSet, b *Bitmap) bool {
+	for i, e := a.NextSet(0); e; i, e = a.NextSet(i + 1) {
+		if !b.ContainsInt(int(i)) {
+			fmt.Printf("in a bitset, not b bitmap, i: %d\n", i)
+			fmt.Printf("  a bitset: %s\n  b bitmap: %s\n",
+				a.String(), b.String())
+			return false
+		}
+	}
+
+	i := b.Iterator()
+	for i.HasNext() {
+		v := i.Next()
+		if !a.Test(uint(v)) {
+			fmt.Printf("in b bitmap, not a bitset, v: %d\n", v)
+			fmt.Printf("  a bitset: %s\n  b bitmap: %s\n",
+				a.String(), b.String())
+			return false
+		}
+	}
+
+	return true
+}

vendor/github.com/RoaringBitmap/roaring/util.go

@@ -0,0 +1,304 @@
+package roaring
+
+import (
+	"math/rand"
+	"sort"
+)
+
+const (
+	arrayDefaultMaxSize        = 4096 // containers with 4096 or fewer integers should be array containers.
+	arrayLazyLowerBound        = 1024
+	maxCapacity                = 1 << 16
+	serialCookieNoRunContainer = 12346 // only arrays and bitmaps
+	invalidCardinality         = -1
+	serialCookie               = 12347 // runs, arrays, and bitmaps
+	noOffsetThreshold          = 4
+
+	// MaxUint32 is the largest uint32 value.
+	MaxUint32 = 4294967295
+
+	// MaxRange is One more than the maximum allowed bitmap bit index. For use as an upper
+	// bound for ranges.
+	MaxRange uint64 = MaxUint32 + 1
+
+	// MaxUint16 is the largest 16 bit unsigned int.
+	// This is the largest value an interval16 can store.
+	MaxUint16 = 65535
+
+	// Compute wordSizeInBytes, the size of a word in bytes.
+	_m              = ^uint64(0)
+	_logS           = _m>>8&1 + _m>>16&1 + _m>>32&1
+	wordSizeInBytes = 1 << _logS
+
+	// other constants used in ctz_generic.go
+	wordSizeInBits = wordSizeInBytes << 3 // word size in bits
+)
+
+const maxWord = 1<<wordSizeInBits - 1
+
+// doesn't apply to runContainers
+func getSizeInBytesFromCardinality(card int) int {
+	if card > arrayDefaultMaxSize {
+		// bitmapContainer
+		return maxCapacity / 8
+	}
+	// arrayContainer
+	return 2 * card
+}
+
+func fill(arr []uint64, val uint64) {
+	for i := range arr {
+		arr[i] = val
+	}
+}
+func fillRange(arr []uint64, start, end int, val uint64) {
+	for i := start; i < end; i++ {
+		arr[i] = val
+	}
+}
+
+func fillArrayAND(container []uint16, bitmap1, bitmap2 []uint64) {
+	if len(bitmap1) != len(bitmap2) {
+		panic("array lengths don't match")
+	}
+	// TODO: rewrite in assembly
+	pos := 0
+	for k := range bitmap1 {
+		bitset := bitmap1[k] & bitmap2[k]
+		for bitset != 0 {
+			t := bitset & -bitset
+			container[pos] = uint16((k*64 + int(popcount(t-1))))
+			pos = pos + 1
+			bitset ^= t
+		}
+	}
+}
+
+func fillArrayANDNOT(container []uint16, bitmap1, bitmap2 []uint64) {
+	if len(bitmap1) != len(bitmap2) {
+		panic("array lengths don't match")
+	}
+	// TODO: rewrite in assembly
+	pos := 0
+	for k := range bitmap1 {
+		bitset := bitmap1[k] &^ bitmap2[k]
+		for bitset != 0 {
+			t := bitset & -bitset
+			container[pos] = uint16((k*64 + int(popcount(t-1))))
+			pos = pos + 1
+			bitset ^= t
+		}
+	}
+}
+
+func fillArrayXOR(container []uint16, bitmap1, bitmap2 []uint64) {
+	if len(bitmap1) != len(bitmap2) {
+		panic("array lengths don't match")
+	}
+	// TODO: rewrite in assembly
+	pos := 0
+	for k := 0; k < len(bitmap1); k++ {
+		bitset := bitmap1[k] ^ bitmap2[k]
+		for bitset != 0 {
+			t := bitset & -bitset
+			container[pos] = uint16((k*64 + int(popcount(t-1))))
+			pos = pos + 1
+			bitset ^= t
+		}
+	}
+}
+
+func highbits(x uint32) uint16 {
+	return uint16(x >> 16)
+}
+func lowbits(x uint32) uint16 {
+	return uint16(x & maxLowBit)
+}
+
+const maxLowBit = 0xFFFF
+
+func flipBitmapRange(bitmap []uint64, start int, end int) {
+	if start >= end {
+		return
+	}
+	firstword := start / 64
+	endword := (end - 1) / 64
+	bitmap[firstword] ^= ^(^uint64(0) << uint(start%64))
+	for i := firstword; i < endword; i++ {
+		bitmap[i] = ^bitmap[i]
+	}
+	bitmap[endword] ^= ^uint64(0) >> (uint(-end) % 64)
+}
+
+func resetBitmapRange(bitmap []uint64, start int, end int) {
+	if start >= end {
+		return
+	}
+	firstword := start / 64
+	endword := (end - 1) / 64
+	if firstword == endword {
+		bitmap[firstword] &= ^((^uint64(0) << uint(start%64)) & (^uint64(0) >> (uint(-end) % 64)))
+		return
+	}
+	bitmap[firstword] &= ^(^uint64(0) << uint(start%64))
+	for i := firstword + 1; i < endword; i++ {
+		bitmap[i] = 0
+	}
+	bitmap[endword] &= ^(^uint64(0) >> (uint(-end) % 64))
+
+}
+
+func setBitmapRange(bitmap []uint64, start int, end int) {
+	if start >= end {
+		return
+	}
+	firstword := start / 64
+	endword := (end - 1) / 64
+	if firstword == endword {
+		bitmap[firstword] |= (^uint64(0) << uint(start%64)) & (^uint64(0) >> (uint(-end) % 64))
+		return
+	}
+	bitmap[firstword] |= ^uint64(0) << uint(start%64)
+	for i := firstword + 1; i < endword; i++ {
+		bitmap[i] = ^uint64(0)
+	}
+	bitmap[endword] |= ^uint64(0) >> (uint(-end) % 64)
+}
+
+func flipBitmapRangeAndCardinalityChange(bitmap []uint64, start int, end int) int {
+	before := wordCardinalityForBitmapRange(bitmap, start, end)
+	flipBitmapRange(bitmap, start, end)
+	after := wordCardinalityForBitmapRange(bitmap, start, end)
+	return int(after - before)
+}
+
+func resetBitmapRangeAndCardinalityChange(bitmap []uint64, start int, end int) int {
+	before := wordCardinalityForBitmapRange(bitmap, start, end)
+	resetBitmapRange(bitmap, start, end)
+	after := wordCardinalityForBitmapRange(bitmap, start, end)
+	return int(after - before)
+}
+
+func setBitmapRangeAndCardinalityChange(bitmap []uint64, start int, end int) int {
+	before := wordCardinalityForBitmapRange(bitmap, start, end)
+	setBitmapRange(bitmap, start, end)
+	after := wordCardinalityForBitmapRange(bitmap, start, end)
+	return int(after - before)
+}
+
+func wordCardinalityForBitmapRange(bitmap []uint64, start int, end int) uint64 {
+	answer := uint64(0)
+	if start >= end {
+		return answer
+	}
+	firstword := start / 64
+	endword := (end - 1) / 64
+	for i := firstword; i <= endword; i++ {
+		answer += popcount(bitmap[i])
+	}
+	return answer
+}
+
+func selectBitPosition(w uint64, j int) int {
+	seen := 0
+
+	// Divide 64bit
+	part := w & 0xFFFFFFFF
+	n := popcount(part)
+	if n <= uint64(j) {
+		part = w >> 32
+		seen += 32
+		j -= int(n)
+	}
+	w = part
+
+	// Divide 32bit
+	part = w & 0xFFFF
+	n = popcount(part)
+	if n <= uint64(j) {
+		part = w >> 16
+		seen += 16
+		j -= int(n)
+	}
+	w = part
+
+	// Divide 16bit
+	part = w & 0xFF
+	n = popcount(part)
+	if n <= uint64(j) {
+		part = w >> 8
+		seen += 8
+		j -= int(n)
+	}
+	w = part
+
+	// Lookup in final byte
+	var counter uint
+	for counter = 0; counter < 8; counter++ {
+		j -= int((w >> counter) & 1)
+		if j < 0 {
+			break
+		}
+	}
+	return seen + int(counter)
+
+}
+
+func panicOn(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+type ph struct {
+	orig int
+	rand int
+}
+
+type pha []ph
+
+func (p pha) Len() int           { return len(p) }
+func (p pha) Less(i, j int) bool { return p[i].rand < p[j].rand }
+func (p pha) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+
+func getRandomPermutation(n int) []int {
+	r := make([]ph, n)
+	for i := 0; i < n; i++ {
+		r[i].orig = i
+		r[i].rand = rand.Intn(1 << 29)
+	}
+	sort.Sort(pha(r))
+	m := make([]int, n)
+	for i := range m {
+		m[i] = r[i].orig
+	}
+	return m
+}
+
+func minOfInt(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}
+
+func maxOfInt(a, b int) int {
+	if a > b {
+		return a
+	}
+	return b
+}
+
+func maxOfUint16(a, b uint16) uint16 {
+	if a > b {
+		return a
+	}
+	return b
+}
+
+func minOfUint16(a, b uint16) uint16 {
+	if a < b {
+		return a
+	}
+	return b
+}

vendor/github.com/blevesearch/bleve/.gitignore

@@ -0,0 +1,19 @@
+#*
+*.sublime-*
+*~
+.#*
+.project
+.settings
+**/.idea/
+**/*.iml
+.DS_Store
+query_string.y.go.tmp
+/analysis/token_filters/cld2/cld2-read-only
+/analysis/token_filters/cld2/libcld2_full.a
+/cmd/bleve/bleve
+vendor/**
+!vendor/manifest
+/y.output
+/search/query/y.output
+*.test
+tags

vendor/github.com/blevesearch/bleve/.travis.yml

@@ -0,0 +1,25 @@
+sudo: false
+
+language: go
+
+go:
+ - "1.12.x"
+ - "1.13.x"
+ - "1.14.x"
+
+script:
+  - go get golang.org/x/tools/cmd/cover
+  - go get github.com/mattn/goveralls
+  - go get github.com/kisielk/errcheck
+  - go get -u github.com/FiloSottile/gvt
+  - gvt restore
+  - go test -race -v $(go list ./... | grep -v vendor/)
+  - go vet $(go list ./... | grep -v vendor/)
+  - go test ./test -v -indexType scorch
+  - errcheck -ignorepkg fmt $(go list ./... | grep -v vendor/);
+  - docs/project-code-coverage.sh
+  - docs/build_children.sh
+
+notifications:
+  email:
+    - marty.schoch@gmail.com

vendor/github.com/blevesearch/bleve/CONTRIBUTING.md

@@ -0,0 +1,16 @@
+# Contributing to Bleve
+
+We look forward to your contributions, but ask that you first review these guidelines.
+
+### Sign the CLA
+
+As Bleve is a Couchbase project we require contributors accept the [Couchbase Contributor License Agreement](http://review.couchbase.org/static/individual_agreement.html). To sign this agreement log into the Couchbase [code review tool](http://review.couchbase.org/). The Bleve project does not use this code review tool but it is still used to track acceptance of the contributor license agreements.
+
+### Submitting a Pull Request
+
+All types of contributions are welcome, but please keep the following in mind:
+
+- If you're planning a large change, you should really discuss it in a github issue or on the google group first. This helps avoid duplicate effort and spending time on something that may not be merged.
+- Existing tests should continue to pass, new tests for the contribution are nice to have.
+- All code should have gone through `go fmt`
+- All code should pass `go vet`

vendor/github.com/blevesearch/bleve/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

vendor/github.com/blevesearch/bleve/README.md

@@ -0,0 +1,67 @@
+# ![bleve](docs/bleve.png) bleve
+
+[![Build Status](https://travis-ci.org/blevesearch/bleve.svg?branch=master)](https://travis-ci.org/blevesearch/bleve) [![Coverage Status](https://coveralls.io/repos/github/blevesearch/bleve/badge.svg?branch=master)](https://coveralls.io/github/blevesearch/bleve?branch=master) [![GoDoc](https://godoc.org/github.com/blevesearch/bleve?status.svg)](https://godoc.org/github.com/blevesearch/bleve)
+[![Join the chat at https://gitter.im/blevesearch/bleve](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/blevesearch/bleve?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+[![codebeat](https://codebeat.co/badges/38a7cbc9-9cf5-41c0-a315-0746178230f4)](https://codebeat.co/projects/github-com-blevesearch-bleve)
+[![Go Report Card](https://goreportcard.com/badge/blevesearch/bleve)](https://goreportcard.com/report/blevesearch/bleve)
+[![Sourcegraph](https://sourcegraph.com/github.com/blevesearch/bleve/-/badge.svg)](https://sourcegraph.com/github.com/blevesearch/bleve?badge)  [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
+
+modern text indexing in go - [blevesearch.com](http://www.blevesearch.com/)
+
+Try out bleve live by [searching the bleve website](http://www.blevesearch.com/search/?q=bleve).
+
+## Features
+
+* Index any go data structure (including JSON)
+* Intelligent defaults backed up by powerful configuration
+* Supported field types:
+    * Text, Numeric, Date
+* Supported query types:
+    * Term, Phrase, Match, Match Phrase, Prefix
+    * Conjunction, Disjunction, Boolean
+    * Numeric Range, Date Range
+    * Simple query [syntax](http://www.blevesearch.com/docs/Query-String-Query/) for human entry
+* tf-idf Scoring
+* Search result match highlighting
+* Supports Aggregating Facets:
+    * Terms Facet
+    * Numeric Range Facet
+    * Date Range Facet
+
+## Discussion
+
+Discuss usage and development of bleve in the [google group](https://groups.google.com/forum/#!forum/bleve).
+
+## Indexing
+
+```go
+message := struct{
+	Id   string
+	From string
+	Body string
+}{
+	Id:   "example",
+	From: "marty.schoch@gmail.com",
+	Body: "bleve indexing is easy",
+}
+
+mapping := bleve.NewIndexMapping()
+index, err := bleve.New("example.bleve", mapping)
+if err != nil {
+	panic(err)
+}
+index.Index(message.Id, message)
+```
+
+## Querying
+
+```go
+index, _ := bleve.Open("example.bleve")
+query := bleve.NewQueryStringQuery("bleve")
+searchRequest := bleve.NewSearchRequest(query)
+searchResult, _ := index.Search(searchRequest)
+```
+
+## License
+
+Apache License Version 2.0

vendor/github.com/blevesearch/bleve/analysis/analyzer/keyword/keyword.go

@@ -0,0 +1,38 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package keyword
+
+import (
+	"github.com/blevesearch/bleve/analysis"
+	"github.com/blevesearch/bleve/analysis/tokenizer/single"
+	"github.com/blevesearch/bleve/registry"
+)
+
+const Name = "keyword"
+
+func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) {
+	keywordTokenizer, err := cache.TokenizerNamed(single.Name)
+	if err != nil {
+		return nil, err
+	}
+	rv := analysis.Analyzer{
+		Tokenizer: keywordTokenizer,
+	}
+	return &rv, nil
+}
+
+func init() {
+	registry.RegisterAnalyzer(Name, AnalyzerConstructor)
+}

vendor/github.com/blevesearch/bleve/analysis/analyzer/standard/standard.go

@@ -0,0 +1,52 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package standard
+
+import (
+	"github.com/blevesearch/bleve/analysis"
+	"github.com/blevesearch/bleve/analysis/lang/en"
+	"github.com/blevesearch/bleve/analysis/token/lowercase"
+	"github.com/blevesearch/bleve/analysis/tokenizer/unicode"
+	"github.com/blevesearch/bleve/registry"
+)
+
+const Name = "standard"
+
+func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) {
+	tokenizer, err := cache.TokenizerNamed(unicode.Name)
+	if err != nil {
+		return nil, err
+	}
+	toLowerFilter, err := cache.TokenFilterNamed(lowercase.Name)
+	if err != nil {
+		return nil, err
+	}
+	stopEnFilter, err := cache.TokenFilterNamed(en.StopName)
+	if err != nil {
+		return nil, err
+	}
+	rv := analysis.Analyzer{
+		Tokenizer: tokenizer,
+		TokenFilters: []analysis.TokenFilter{
+			toLowerFilter,
+			stopEnFilter,
+		},
+	}
+	return &rv, nil
+}
+
+func init() {
+	registry.RegisterAnalyzer(Name, AnalyzerConstructor)
+}

vendor/github.com/blevesearch/bleve/analysis/datetime/flexible/flexible.go

@@ -0,0 +1,64 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package flexible
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/blevesearch/bleve/analysis"
+	"github.com/blevesearch/bleve/registry"
+)
+
+const Name = "flexiblego"
+
+type DateTimeParser struct {
+	layouts []string
+}
+
+func New(layouts []string) *DateTimeParser {
+	return &DateTimeParser{
+		layouts: layouts,
+	}
+}
+
+func (p *DateTimeParser) ParseDateTime(input string) (time.Time, error) {
+	for _, layout := range p.layouts {
+		rv, err := time.Parse(layout, input)
+		if err == nil {
+			return rv, nil
+		}
+	}
+	return time.Time{}, analysis.ErrInvalidDateTime
+}
+
+func DateTimeParserConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.DateTimeParser, error) {
+	layouts, ok := config["layouts"].([]interface{})
+	if !ok {
+		return nil, fmt.Errorf("must specify layouts")
+	}
+	var layoutStrs []string
+	for _, layout := range layouts {
+		layoutStr, ok := layout.(string)
+		if ok {
+			layoutStrs = append(layoutStrs, layoutStr)
+		}
+	}
+	return New(layoutStrs), nil
+}
+
+func init() {
+	registry.RegisterDateTimeParser(Name, DateTimeParserConstructor)
+}

vendor/github.com/blevesearch/bleve/analysis/datetime/optional/optional.go

@@ -0,0 +1,45 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package optional
+
+import (
+	"time"
+
+	"github.com/blevesearch/bleve/analysis"
+	"github.com/blevesearch/bleve/analysis/datetime/flexible"
+	"github.com/blevesearch/bleve/registry"
+)
+
+const Name = "dateTimeOptional"
+
+const rfc3339NoTimezone = "2006-01-02T15:04:05"
+const rfc3339NoTimezoneNoT = "2006-01-02 15:04:05"
+const rfc3339NoTime = "2006-01-02"
+
+var layouts = []string{
+	time.RFC3339Nano,
+	time.RFC3339,
+	rfc3339NoTimezone,
+	rfc3339NoTimezoneNoT,
+	rfc3339NoTime,
+}
+
+func DateTimeParserConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.DateTimeParser, error) {
+	return flexible.New(layouts), nil
+}
+
+func init() {
+	registry.RegisterDateTimeParser(Name, DateTimeParserConstructor)
+}

vendor/github.com/blevesearch/bleve/analysis/freq.go

@@ -0,0 +1,152 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package analysis
+
+import (
+	"reflect"
+
+	"github.com/blevesearch/bleve/size"
+)
+
+var reflectStaticSizeTokenLocation int
+var reflectStaticSizeTokenFreq int
+
+func init() {
+	var tl TokenLocation
+	reflectStaticSizeTokenLocation = int(reflect.TypeOf(tl).Size())
+	var tf TokenFreq
+	reflectStaticSizeTokenFreq = int(reflect.TypeOf(tf).Size())
+}
+
+// TokenLocation represents one occurrence of a term at a particular location in
+// a field. Start, End and Position have the same meaning as in analysis.Token.
+// Field and ArrayPositions identify the field value in the source document.
+// See document.Field for details.
+type TokenLocation struct {
+	Field          string
+	ArrayPositions []uint64
+	Start          int
+	End            int
+	Position       int
+}
+
+func (tl *TokenLocation) Size() int {
+	rv := reflectStaticSizeTokenLocation
+	rv += len(tl.ArrayPositions) * size.SizeOfUint64
+	return rv
+}
+
+// TokenFreq represents all the occurrences of a term in all fields of a
+// document.
+type TokenFreq struct {
+	Term      []byte
+	Locations []*TokenLocation
+	frequency int
+}
+
+func (tf *TokenFreq) Size() int {
+	rv := reflectStaticSizeTokenFreq
+	rv += len(tf.Term)
+	for _, loc := range tf.Locations {
+		rv += loc.Size()
+	}
+	return rv
+}
+
+func (tf *TokenFreq) Frequency() int {
+	return tf.frequency
+}
+
+// TokenFrequencies maps document terms to their combined frequencies from all
+// fields.
+type TokenFrequencies map[string]*TokenFreq
+
+func (tfs TokenFrequencies) Size() int {
+	rv := size.SizeOfMap
+	rv += len(tfs) * (size.SizeOfString + size.SizeOfPtr)
+	for k, v := range tfs {
+		rv += len(k)
+		rv += v.Size()
+	}
+	return rv
+}
+
+func (tfs TokenFrequencies) MergeAll(remoteField string, other TokenFrequencies) {
+	// walk the new token frequencies
+	for tfk, tf := range other {
+		// set the remoteField value in incoming token freqs
+		for _, l := range tf.Locations {
+			l.Field = remoteField
+		}
+		existingTf, exists := tfs[tfk]
+		if exists {
+			existingTf.Locations = append(existingTf.Locations, tf.Locations...)
+			existingTf.frequency = existingTf.frequency + tf.frequency
+		} else {
+			tfs[tfk] = &TokenFreq{
+				Term:      tf.Term,
+				frequency: tf.frequency,
+				Locations: make([]*TokenLocation, len(tf.Locations)),
+			}
+			copy(tfs[tfk].Locations, tf.Locations)
+		}
+	}
+}
+
+func TokenFrequency(tokens TokenStream, arrayPositions []uint64, includeTermVectors bool) TokenFrequencies {
+	rv := make(map[string]*TokenFreq, len(tokens))
+
+	if includeTermVectors {
+		tls := make([]TokenLocation, len(tokens))
+		tlNext := 0
+
+		for _, token := range tokens {
+			tls[tlNext] = TokenLocation{
+				ArrayPositions: arrayPositions,
+				Start:          token.Start,
+				End:            token.End,
+				Position:       token.Position,
+			}
+
+			curr, ok := rv[string(token.Term)]
+			if ok {
+				curr.Locations = append(curr.Locations, &tls[tlNext])
+				curr.frequency++
+			} else {
+				rv[string(token.Term)] = &TokenFreq{
+					Term:      token.Term,
+					Locations: []*TokenLocation{&tls[tlNext]},
+					frequency: 1,
+				}
+			}
+
+			tlNext++
+		}
+	} else {
+		for _, token := range tokens {
+			curr, exists := rv[string(token.Term)]
+			if exists {
+				curr.frequency++
+			} else {
+				rv[string(token.Term)] = &TokenFreq{
+					Term:      token.Term,
+					frequency: 1,
+				}
+			}
+		}
+	}
+
+	return rv
+}

vendor/github.com/blevesearch/bleve/analysis/lang/en/analyzer_en.go

@@ -0,0 +1,70 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package en implements an analyzer with reasonable defaults for processing
+// English text.
+//
+// It strips possessive suffixes ('s), transforms tokens to lower case,
+// removes stopwords from a built-in list, and applies porter stemming.
+//
+// The built-in stopwords list is defined in EnglishStopWords.
+package en
+
+import (
+	"github.com/blevesearch/bleve/analysis"
+	"github.com/blevesearch/bleve/registry"
+
+	"github.com/blevesearch/bleve/analysis/token/lowercase"
+	"github.com/blevesearch/bleve/analysis/token/porter"
+	"github.com/blevesearch/bleve/analysis/tokenizer/unicode"
+)
+
+const AnalyzerName = "en"
+
+func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) {
+	tokenizer, err := cache.TokenizerNamed(unicode.Name)
+	if err != nil {
+		return nil, err
+	}
+	possEnFilter, err := cache.TokenFilterNamed(PossessiveName)
+	if err != nil {
+		return nil, err
+	}
+	toLowerFilter, err := cache.TokenFilterNamed(lowercase.Name)
+	if err != nil {
+		return nil, err
+	}
+	stopEnFilter, err := cache.TokenFilterNamed(StopName)
+	if err != nil {
+		return nil, err
+	}
+	stemmerEnFilter, err := cache.TokenFilterNamed(porter.Name)
+	if err != nil {
+		return nil, err
+	}
+	rv := analysis.Analyzer{
+		Tokenizer: tokenizer,
+		TokenFilters: []analysis.TokenFilter{
+			possEnFilter,
+			toLowerFilter,
+			stopEnFilter,
+			stemmerEnFilter,
+		},
+	}
+	return &rv, nil
+}
+
+func init() {
+	registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor)
+}

vendor/github.com/blevesearch/bleve/analysis/lang/en/possessive_filter_en.go

@@ -0,0 +1,67 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package en
+
+import (
+	"unicode/utf8"
+
+	"github.com/blevesearch/bleve/analysis"
+	"github.com/blevesearch/bleve/registry"
+)
+
+// PossessiveName is the name PossessiveFilter is registered as
+// in the bleve registry.
+const PossessiveName = "possessive_en"
+
+const rightSingleQuotationMark = '’'
+const apostrophe = '\''
+const fullWidthApostrophe = '＇'
+
+const apostropheChars = rightSingleQuotationMark + apostrophe + fullWidthApostrophe
+
+// PossessiveFilter implements a TokenFilter which
+// strips the English possessive suffix ('s) from tokens.
+// It handle a variety of apostrophe types, is case-insensitive
+// and doesn't distinguish between possessive and contraction.
+// (ie "She's So Rad" becomes "She So Rad")
+type PossessiveFilter struct {
+}
+
+func NewPossessiveFilter() *PossessiveFilter {
+	return &PossessiveFilter{}
+}
+
+func (s *PossessiveFilter) Filter(input analysis.TokenStream) analysis.TokenStream {
+	for _, token := range input {
+		lastRune, lastRuneSize := utf8.DecodeLastRune(token.Term)
+		if lastRune == 's' || lastRune == 'S' {
+			nextLastRune, nextLastRuneSize := utf8.DecodeLastRune(token.Term[:len(token.Term)-lastRuneSize])
+			if nextLastRune == rightSingleQuotationMark ||
+				nextLastRune == apostrophe ||
+				nextLastRune == fullWidthApostrophe {
+				token.Term = token.Term[:len(token.Term)-lastRuneSize-nextLastRuneSize]
+			}
+		}
+	}
+	return input
+}
+
+func PossessiveFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {
+	return NewPossessiveFilter(), nil
+}
+
+func init() {
+	registry.RegisterTokenFilter(PossessiveName, PossessiveFilterConstructor)
+}

vendor/github.com/blevesearch/bleve/analysis/lang/en/stemmer_en_snowball.go

@@ -0,0 +1,49 @@
+//  Copyright (c) 2020 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package en
+
+import (
+	"github.com/blevesearch/bleve/analysis"
+	"github.com/blevesearch/bleve/registry"
+
+	"github.com/blevesearch/snowballstem"
+	"github.com/blevesearch/snowballstem/english"
+)
+
+const SnowballStemmerName = "stemmer_en_snowball"
+
+type EnglishStemmerFilter struct {
+}
+
+func NewEnglishStemmerFilter() *EnglishStemmerFilter {
+	return &EnglishStemmerFilter{}
+}
+
+func (s *EnglishStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream {
+	for _, token := range input {
+		env := snowballstem.NewEnv(string(token.Term))
+		english.Stem(env)
+		token.Term = []byte(env.Current())
+	}
+	return input
+}
+
+func EnglishStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {
+	return NewEnglishStemmerFilter(), nil
+}
+
+func init() {
+	registry.RegisterTokenFilter(SnowballStemmerName, EnglishStemmerFilterConstructor)
+}

vendor/github.com/blevesearch/bleve/analysis/lang/en/stop_filter_en.go

@@ -0,0 +1,33 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package en
+
+import (
+	"github.com/blevesearch/bleve/analysis"
+	"github.com/blevesearch/bleve/analysis/token/stop"
+	"github.com/blevesearch/bleve/registry"
+)
+
+func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {
+	tokenMap, err := cache.TokenMapNamed(StopName)
+	if err != nil {
+		return nil, err
+	}
+	return stop.NewStopTokensFilter(tokenMap), nil
+}
+
+func init() {
+	registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor)
+}

vendor/github.com/blevesearch/bleve/analysis/lang/en/stop_words_en.go

@@ -0,0 +1,344 @@
+package en
+
+import (
+	"github.com/blevesearch/bleve/analysis"
+	"github.com/blevesearch/bleve/registry"
+)
+
+const StopName = "stop_en"
+
+// EnglishStopWords is the built-in list of stopwords used by the "stop_en" TokenFilter.
+//
+// this content was obtained from:
+// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/
+// ` was changed to ' to allow for literal string
+var EnglishStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/english/stop.txt
+ | This file is distributed under the BSD License.
+ | See http://snowball.tartarus.org/license.php
+ | Also see http://www.opensource.org/licenses/bsd-license.html
+ |  - Encoding was converted to UTF-8.
+ |  - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
+ 
+ | An English stop word list. Comments begin with vertical bar. Each stop
+ | word is at the start of a line.
+
+ | Many of the forms below are quite rare (e.g. "yourselves") but included for
+ |  completeness.
+
+           | PRONOUNS FORMS
+             | 1st person sing
+
+i              | subject, always in upper case of course
+
+me             | object
+my             | possessive adjective
+               | the possessive pronoun 'mine' is best suppressed, because of the
+               | sense of coal-mine etc.
+myself         | reflexive
+             | 1st person plural
+we             | subject
+
+| us           | object
+               | care is required here because US = United States. It is usually
+               | safe to remove it if it is in lower case.
+our            | possessive adjective
+ours           | possessive pronoun
+ourselves      | reflexive
+             | second person (archaic 'thou' forms not included)
+you            | subject and object
+your           | possessive adjective
+yours          | possessive pronoun
+yourself       | reflexive (singular)
+yourselves     | reflexive (plural)
+             | third person singular
+he             | subject
+him            | object
+his            | possessive adjective and pronoun
+himself        | reflexive
+
+she            | subject
+her            | object and possessive adjective
+hers           | possessive pronoun
+herself        | reflexive
+
+it             | subject and object
+its            | possessive adjective
+itself         | reflexive
+             | third person plural
+they           | subject
+them           | object
+their          | possessive adjective
+theirs         | possessive pronoun
+themselves     | reflexive
+             | other forms (demonstratives, interrogatives)
+what
+which
+who
+whom
+this
+that
+these
+those
+
+           | VERB FORMS (using F.R. Palmer's nomenclature)
+             | BE
+am             | 1st person, present
+is             | -s form (3rd person, present)
+are            | present
+was            | 1st person, past
+were           | past
+be             | infinitive
+been           | past participle
+being          | -ing form
+             | HAVE
+have           | simple
+has            | -s form
+had            | past
+having         | -ing form
+             | DO
+do             | simple
+does           | -s form
+did            | past
+doing          | -ing form
+
+ | The forms below are, I believe, best omitted, because of the significant
+ | homonym forms:
+
+ |  He made a WILL
+ |  old tin CAN
+ |  merry month of MAY
+ |  a smell of MUST
+ |  fight the good fight with all thy MIGHT
+
+ | would, could, should, ought might however be included
+
+ |          | AUXILIARIES
+ |            | WILL
+ |will
+
+would
+
+ |            | SHALL
+ |shall
+
+should
+
+ |            | CAN
+ |can
+
+could
+
+ |            | MAY
+ |may
+ |might
+ |            | MUST
+ |must
+ |            | OUGHT
+
+ought
+
+           | COMPOUND FORMS, increasingly encountered nowadays in 'formal' writing
+              | pronoun + verb
+
+i'm
+you're
+he's
+she's
+it's
+we're
+they're
+i've
+you've
+we've
+they've
+i'd
+you'd
+he'd
+she'd
+we'd
+they'd
+i'll
+you'll
+he'll
+she'll
+we'll
+they'll
+
+              | verb + negation
+
+isn't
+aren't
+wasn't
+weren't
+hasn't
+haven't
+hadn't
+doesn't
+don't
+didn't
+
+              | auxiliary + negation
+
+won't
+wouldn't
+shan't
+shouldn't
+can't
+cannot
+couldn't
+mustn't
+
+             | miscellaneous forms
+
+let's
+that's
+who's
+what's
+here's
+there's
+when's
+where's
+why's
+how's
+
+              | rarer forms
+
+ | daren't needn't
+
+              | doubtful forms
+
+ | oughtn't mightn't
+
+           | ARTICLES
+a
+an
+the
+
+           | THE REST (Overlap among prepositions, conjunctions, adverbs etc is so
+           | high, that classification is pointless.)
+and
+but
+if
+or
+because
+as
+until
+while
+
+of
+at
+by
+for
+with
+about
+against
+between
+into
+through
+during
+before
+after
+above
+below
+to
+from
+up
+down
+in
+out
+on
+off
+over
+under
+
+again
+further
+then
+once
+
+here
+there
+when
+where
+why
+how
+
+all
+any
+both
+each
+few
+more
+most
+other
+some
+such
+
+no
+nor
+not
+only
+own
+same
+so
+than
+too
+very
+
+ | Just for the record, the following words are among the commonest in English
+
+    | one
+    | every
+    | least
+    | less
+    | many
+    | now
+    | ever
+    | never
+    | say
+    | says
+    | said
+    | also
+    | get
+    | go
+    | goes
+    | just
+    | made
+    | make
+    | put
+    | see
+    | seen
+    | whether
+    | like
+    | well
+    | back
+    | even
+    | still
+    | way
+    | take
+    | since
+    | another
+    | however
+    | two
+    | three
+    | four
+    | five
+    | first
+    | second
+    | new
+    | old
+    | high
+    | long
+`)
+
+func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {
+	rv := analysis.NewTokenMap()
+	err := rv.LoadBytes(EnglishStopWords)
+	return rv, err
+}
+
+func init() {
+	registry.RegisterTokenMap(StopName, TokenMapConstructor)
+}

vendor/github.com/blevesearch/bleve/analysis/test_words.txt

@@ -0,0 +1,7 @@
+# full line comment
+marty
+steve # trailing comment
+| different format of comment
+dustin
+siri | different style trailing comment
+multiple words	with different	whitespace

vendor/github.com/blevesearch/bleve/analysis/token/lowercase/lowercase.go

@@ -0,0 +1,105 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package lowercase implements a TokenFilter which converts
+// tokens to lower case according to unicode rules.
+package lowercase
+
+import (
+	"bytes"
+	"unicode"
+	"unicode/utf8"
+
+	"github.com/blevesearch/bleve/analysis"
+	"github.com/blevesearch/bleve/registry"
+)
+
+// Name is the name used to register LowerCaseFilter in the bleve registry
+const Name = "to_lower"
+
+type LowerCaseFilter struct {
+}
+
+func NewLowerCaseFilter() *LowerCaseFilter {
+	return &LowerCaseFilter{}
+}
+
+func (f *LowerCaseFilter) Filter(input analysis.TokenStream) analysis.TokenStream {
+	for _, token := range input {
+		token.Term = toLowerDeferredCopy(token.Term)
+	}
+	return input
+}
+
+func LowerCaseFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {
+	return NewLowerCaseFilter(), nil
+}
+
+func init() {
+	registry.RegisterTokenFilter(Name, LowerCaseFilterConstructor)
+}
+
+// toLowerDeferredCopy will function exactly like
+// bytes.ToLower() only it will reuse (overwrite)
+// the original byte array when possible
+// NOTE: because its possible that the lower-case
+// form of a rune has a different utf-8 encoded
+// length, in these cases a new byte array is allocated
+func toLowerDeferredCopy(s []byte) []byte {
+	j := 0
+	for i := 0; i < len(s); {
+		wid := 1
+		r := rune(s[i])
+		if r >= utf8.RuneSelf {
+			r, wid = utf8.DecodeRune(s[i:])
+		}
+
+		l := unicode.ToLower(r)
+
+		// If the rune is already lowercased, just move to the
+		// next rune.
+		if l == r {
+			i += wid
+			j += wid
+			continue
+		}
+
+		// Handles the Unicode edge-case where the last
+		// rune in a word on the greek Σ needs to be converted
+		// differently.
+		if l == 'σ' && i+2 == len(s) {
+			l = 'ς'
+		}
+
+		lwid := utf8.RuneLen(l)
+		if lwid > wid {
+			// utf-8 encoded replacement is wider
+			// for now, punt and defer
+			// to bytes.ToLower() for the remainder
+			// only known to happen with chars
+			//   Rune Ⱥ(570) width 2 - Lower ⱥ(11365) width 3
+			//   Rune Ⱦ(574) width 2 - Lower ⱦ(11366) width 3
+			rest := bytes.ToLower(s[i:])
+			rv := make([]byte, j+len(rest))
+			copy(rv[:j], s[:j])
+			copy(rv[j:], rest)
+			return rv
+		} else {
+			utf8.EncodeRune(s[j:], l)
+		}
+		i += wid
+		j += lwid
+	}
+	return s[:j]
+}

vendor/github.com/blevesearch/bleve/analysis/token/porter/porter.go

@@ -0,0 +1,53 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package porter
+
+import (
+	"bytes"
+
+	"github.com/blevesearch/bleve/analysis"
+	"github.com/blevesearch/bleve/registry"
+
+	"github.com/blevesearch/go-porterstemmer"
+)
+
+const Name = "stemmer_porter"
+
+type PorterStemmer struct {
+}
+
+func NewPorterStemmer() *PorterStemmer {
+	return &PorterStemmer{}
+}
+
+func (s *PorterStemmer) Filter(input analysis.TokenStream) analysis.TokenStream {
+	for _, token := range input {
+		// if it is not a protected keyword, stem it
+		if !token.KeyWord {
+			termRunes := bytes.Runes(token.Term)
+			stemmedRunes := porterstemmer.StemWithoutLowerCasing(termRunes)
+			token.Term = analysis.BuildTermFromRunes(stemmedRunes)
+		}
+	}
+	return input
+}
+
+func PorterStemmerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {
+	return NewPorterStemmer(), nil
+}
+
+func init() {
+	registry.RegisterTokenFilter(Name, PorterStemmerConstructor)
+}

vendor/github.com/blevesearch/bleve/analysis/token/stop/stop.go

@@ -0,0 +1,70 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package stop implements a TokenFilter removing tokens found in
+// a TokenMap.
+//
+// It constructor takes the following arguments:
+//
+// "stop_token_map" (string): the name of the token map identifying tokens to
+// remove.
+package stop
+
+import (
+	"fmt"
+
+	"github.com/blevesearch/bleve/analysis"
+	"github.com/blevesearch/bleve/registry"
+)
+
+const Name = "stop_tokens"
+
+type StopTokensFilter struct {
+	stopTokens analysis.TokenMap
+}
+
+func NewStopTokensFilter(stopTokens analysis.TokenMap) *StopTokensFilter {
+	return &StopTokensFilter{
+		stopTokens: stopTokens,
+	}
+}
+
+func (f *StopTokensFilter) Filter(input analysis.TokenStream) analysis.TokenStream {
+	j := 0
+	for _, token := range input {
+		_, isStopToken := f.stopTokens[string(token.Term)]
+		if !isStopToken {
+			input[j] = token
+			j++
+		}
+	}
+
+	return input[:j]
+}
+
+func StopTokensFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {
+	stopTokenMapName, ok := config["stop_token_map"].(string)
+	if !ok {
+		return nil, fmt.Errorf("must specify stop_token_map")
+	}
+	stopTokenMap, err := cache.TokenMapNamed(stopTokenMapName)
+	if err != nil {
+		return nil, fmt.Errorf("error building stop words filter: %v", err)
+	}
+	return NewStopTokensFilter(stopTokenMap), nil
+}
+
+func init() {
+	registry.RegisterTokenFilter(Name, StopTokensFilterConstructor)
+}

vendor/github.com/blevesearch/bleve/analysis/tokenizer/single/single.go

@@ -0,0 +1,49 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package single
+
+import (
+	"github.com/blevesearch/bleve/analysis"
+	"github.com/blevesearch/bleve/registry"
+)
+
+const Name = "single"
+
+type SingleTokenTokenizer struct {
+}
+
+func NewSingleTokenTokenizer() *SingleTokenTokenizer {
+	return &SingleTokenTokenizer{}
+}
+
+func (t *SingleTokenTokenizer) Tokenize(input []byte) analysis.TokenStream {
+	return analysis.TokenStream{
+		&analysis.Token{
+			Term:     input,
+			Position: 1,
+			Start:    0,
+			End:      len(input),
+			Type:     analysis.AlphaNumeric,
+		},
+	}
+}
+
+func SingleTokenTokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) {
+	return NewSingleTokenTokenizer(), nil
+}
+
+func init() {
+	registry.RegisterTokenizer(Name, SingleTokenTokenizerConstructor)
+}

vendor/github.com/blevesearch/bleve/analysis/tokenizer/unicode/unicode.go

@@ -0,0 +1,131 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package unicode
+
+import (
+	"github.com/blevesearch/segment"
+
+	"github.com/blevesearch/bleve/analysis"
+	"github.com/blevesearch/bleve/registry"
+)
+
+const Name = "unicode"
+
+type UnicodeTokenizer struct {
+}
+
+func NewUnicodeTokenizer() *UnicodeTokenizer {
+	return &UnicodeTokenizer{}
+}
+
+func (rt *UnicodeTokenizer) Tokenize(input []byte) analysis.TokenStream {
+	rvx := make([]analysis.TokenStream, 0, 10) // When rv gets full, append to rvx.
+	rv := make(analysis.TokenStream, 0, 1)
+
+	ta := []analysis.Token(nil)
+	taNext := 0
+
+	segmenter := segment.NewWordSegmenterDirect(input)
+	start := 0
+	pos := 1
+
+	guessRemaining := func(end int) int {
+		avgSegmentLen := end / (len(rv) + 1)
+		if avgSegmentLen < 1 {
+			avgSegmentLen = 1
+		}
+
+		remainingLen := len(input) - end
+
+		return remainingLen / avgSegmentLen
+	}
+
+	for segmenter.Segment() {
+		segmentBytes := segmenter.Bytes()
+		end := start + len(segmentBytes)
+		if segmenter.Type() != segment.None {
+			if taNext >= len(ta) {
+				remainingSegments := guessRemaining(end)
+				if remainingSegments > 1000 {
+					remainingSegments = 1000
+				}
+				if remainingSegments < 1 {
+					remainingSegments = 1
+				}
+
+				ta = make([]analysis.Token, remainingSegments)
+				taNext = 0
+			}
+
+			token := &ta[taNext]
+			taNext++
+
+			token.Term = segmentBytes
+			token.Start = start
+			token.End = end
+			token.Position = pos
+			token.Type = convertType(segmenter.Type())
+
+			if len(rv) >= cap(rv) { // When rv is full, save it into rvx.
+				rvx = append(rvx, rv)
+
+				rvCap := cap(rv) * 2
+				if rvCap > 256 {
+					rvCap = 256
+				}
+
+				rv = make(analysis.TokenStream, 0, rvCap) // Next rv cap is bigger.
+			}
+
+			rv = append(rv, token)
+			pos++
+		}
+		start = end
+	}
+
+	if len(rvx) > 0 {
+		n := len(rv)
+		for _, r := range rvx {
+			n += len(r)
+		}
+		rall := make(analysis.TokenStream, 0, n)
+		for _, r := range rvx {
+			rall = append(rall, r...)
+		}
+		return append(rall, rv...)
+	}
+
+	return rv
+}
+
+func UnicodeTokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) {
+	return NewUnicodeTokenizer(), nil
+}
+
+func init() {
+	registry.RegisterTokenizer(Name, UnicodeTokenizerConstructor)
+}
+
+func convertType(segmentWordType int) analysis.TokenType {
+	switch segmentWordType {
+	case segment.Ideo:
+		return analysis.Ideographic
+	case segment.Kana:
+		return analysis.Ideographic
+	case segment.Number:
+		return analysis.Numeric
+	}
+	return analysis.AlphaNumeric
+}

vendor/github.com/blevesearch/bleve/analysis/tokenmap.go

@@ -0,0 +1,76 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package analysis
+
+import (
+	"bufio"
+	"bytes"
+	"io"
+	"io/ioutil"
+	"strings"
+)
+
+type TokenMap map[string]bool
+
+func NewTokenMap() TokenMap {
+	return make(TokenMap, 0)
+}
+
+// LoadFile reads in a list of tokens from a text file,
+// one per line.
+// Comments are supported using `#` or `|`
+func (t TokenMap) LoadFile(filename string) error {
+	data, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return err
+	}
+	return t.LoadBytes(data)
+}
+
+// LoadBytes reads in a list of tokens from memory,
+// one per line.
+// Comments are supported using `#` or `|`
+func (t TokenMap) LoadBytes(data []byte) error {
+	bytesReader := bytes.NewReader(data)
+	bufioReader := bufio.NewReader(bytesReader)
+	line, err := bufioReader.ReadString('\n')
+	for err == nil {
+		t.LoadLine(line)
+		line, err = bufioReader.ReadString('\n')
+	}
+	// if the err was EOF we still need to process the last value
+	if err == io.EOF {
+		t.LoadLine(line)
+		return nil
+	}
+	return err
+}
+
+func (t TokenMap) LoadLine(line string) {
+	// find the start of a comment, if any
+	startComment := strings.IndexAny(line, "#|")
+	if startComment >= 0 {
+		line = line[:startComment]
+	}
+
+	tokens := strings.Fields(line)
+	for _, token := range tokens {
+		t.AddToken(token)
+	}
+}
+
+func (t TokenMap) AddToken(token string) {
+	t[token] = true
+}

vendor/github.com/blevesearch/bleve/analysis/type.go

@@ -0,0 +1,103 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package analysis
+
+import (
+	"fmt"
+	"time"
+)
+
+type CharFilter interface {
+	Filter([]byte) []byte
+}
+
+type TokenType int
+
+const (
+	AlphaNumeric TokenType = iota
+	Ideographic
+	Numeric
+	DateTime
+	Shingle
+	Single
+	Double
+	Boolean
+)
+
+// Token represents one occurrence of a term at a particular location in a
+// field.
+type Token struct {
+	// Start specifies the byte offset of the beginning of the term in the
+	// field.
+	Start int `json:"start"`
+
+	// End specifies the byte offset of the end of the term in the field.
+	End  int    `json:"end"`
+	Term []byte `json:"term"`
+
+	// Position specifies the 1-based index of the token in the sequence of
+	// occurrences of its term in the field.
+	Position int       `json:"position"`
+	Type     TokenType `json:"type"`
+	KeyWord  bool      `json:"keyword"`
+}
+
+func (t *Token) String() string {
+	return fmt.Sprintf("Start: %d  End: %d  Position: %d  Token: %s  Type: %d", t.Start, t.End, t.Position, string(t.Term), t.Type)
+}
+
+type TokenStream []*Token
+
+// A Tokenizer splits an input string into tokens, the usual behaviour being to
+// map words to tokens.
+type Tokenizer interface {
+	Tokenize([]byte) TokenStream
+}
+
+// A TokenFilter adds, transforms or removes tokens from a token stream.
+type TokenFilter interface {
+	Filter(TokenStream) TokenStream
+}
+
+type Analyzer struct {
+	CharFilters  []CharFilter
+	Tokenizer    Tokenizer
+	TokenFilters []TokenFilter
+}
+
+func (a *Analyzer) Analyze(input []byte) TokenStream {
+	if a.CharFilters != nil {
+		for _, cf := range a.CharFilters {
+			input = cf.Filter(input)
+		}
+	}
+	tokens := a.Tokenizer.Tokenize(input)
+	if a.TokenFilters != nil {
+		for _, tf := range a.TokenFilters {
+			tokens = tf.Filter(tokens)
+		}
+	}
+	return tokens
+}
+
+var ErrInvalidDateTime = fmt.Errorf("unable to parse datetime with any of the layouts")
+
+type DateTimeParser interface {
+	ParseDateTime(string) (time.Time, error)
+}
+
+type ByteArrayConverter interface {
+	Convert([]byte) (interface{}, error)
+}

vendor/github.com/blevesearch/bleve/analysis/util.go

@@ -0,0 +1,92 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package analysis
+
+import (
+	"bytes"
+	"unicode/utf8"
+)
+
+func DeleteRune(in []rune, pos int) []rune {
+	if pos >= len(in) {
+		return in
+	}
+	copy(in[pos:], in[pos+1:])
+	return in[:len(in)-1]
+}
+
+func InsertRune(in []rune, pos int, r rune) []rune {
+	// create a new slice 1 rune larger
+	rv := make([]rune, len(in)+1)
+	// copy the characters before the insert pos
+	copy(rv[0:pos], in[0:pos])
+	// set the inserted rune
+	rv[pos] = r
+	// copy the characters after the insert pos
+	copy(rv[pos+1:], in[pos:])
+	return rv
+}
+
+// BuildTermFromRunesOptimistic will build a term from the provided runes
+// AND optimistically attempt to encode into the provided buffer
+// if at any point it appears the buffer is too small, a new buffer is
+// allocated and that is used instead
+// this should be used in cases where frequently the new term is the same
+// length or shorter than the original term (in number of bytes)
+func BuildTermFromRunesOptimistic(buf []byte, runes []rune) []byte {
+	rv := buf
+	used := 0
+	for _, r := range runes {
+		nextLen := utf8.RuneLen(r)
+		if used+nextLen > len(rv) {
+			// alloc new buf
+			buf = make([]byte, len(runes)*utf8.UTFMax)
+			// copy work we've already done
+			copy(buf, rv[:used])
+			rv = buf
+		}
+		written := utf8.EncodeRune(rv[used:], r)
+		used += written
+	}
+	return rv[:used]
+}
+
+func BuildTermFromRunes(runes []rune) []byte {
+	return BuildTermFromRunesOptimistic(make([]byte, len(runes)*utf8.UTFMax), runes)
+}
+
+func TruncateRunes(input []byte, num int) []byte {
+	runes := bytes.Runes(input)
+	runes = runes[:len(runes)-num]
+	out := BuildTermFromRunes(runes)
+	return out
+}
+
+func RunesEndsWith(input []rune, suffix string) bool {
+	inputLen := len(input)
+	suffixRunes := []rune(suffix)
+	suffixLen := len(suffixRunes)
+	if suffixLen > inputLen {
+		return false
+	}
+
+	for i := suffixLen - 1; i >= 0; i-- {
+		if input[inputLen-(suffixLen-i)] != suffixRunes[i] {
+			return false
+		}
+	}
+
+	return true
+}

vendor/github.com/blevesearch/bleve/config.go

@@ -0,0 +1,91 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bleve
+
+import (
+	"expvar"
+	"io/ioutil"
+	"log"
+	"time"
+
+	"github.com/blevesearch/bleve/index"
+	"github.com/blevesearch/bleve/index/store/gtreap"
+	"github.com/blevesearch/bleve/index/upsidedown"
+	"github.com/blevesearch/bleve/registry"
+	"github.com/blevesearch/bleve/search/highlight/highlighter/html"
+
+	// force import of scorch so its accessible by default
+	_ "github.com/blevesearch/bleve/index/scorch"
+)
+
+var bleveExpVar = expvar.NewMap("bleve")
+
+type configuration struct {
+	Cache                  *registry.Cache
+	DefaultHighlighter     string
+	DefaultKVStore         string
+	DefaultMemKVStore      string
+	DefaultIndexType       string
+	SlowSearchLogThreshold time.Duration
+	analysisQueue          *index.AnalysisQueue
+}
+
+func (c *configuration) SetAnalysisQueueSize(n int) {
+	c.analysisQueue = index.NewAnalysisQueue(n)
+}
+
+func newConfiguration() *configuration {
+	return &configuration{
+		Cache:         registry.NewCache(),
+		analysisQueue: index.NewAnalysisQueue(4),
+	}
+}
+
+// Config contains library level configuration
+var Config *configuration
+
+func init() {
+	bootStart := time.Now()
+
+	// build the default configuration
+	Config = newConfiguration()
+
+	// set the default highlighter
+	Config.DefaultHighlighter = html.Name
+
+	// default kv store
+	Config.DefaultKVStore = ""
+
+	// default mem only kv store
+	Config.DefaultMemKVStore = gtreap.Name
+
+	// default index
+	Config.DefaultIndexType = upsidedown.Name
+
+	bootDuration := time.Since(bootStart)
+	bleveExpVar.Add("bootDuration", int64(bootDuration))
+	indexStats = NewIndexStats()
+	bleveExpVar.Set("indexes", indexStats)
+
+	initDisk()
+}
+
+var logger = log.New(ioutil.Discard, "bleve", log.LstdFlags)
+
+// SetLog sets the logger used for logging
+// by default log messages are sent to ioutil.Discard
+func SetLog(l *log.Logger) {
+	logger = l
+}

vendor/github.com/blevesearch/bleve/config_app.go

@@ -0,0 +1,23 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build appengine appenginevm
+
+package bleve
+
+// in the appengine environment we cannot support disk based indexes
+// so we do no extra configuration in this method
+func initDisk() {
+
+}

vendor/github.com/blevesearch/bleve/config_disk.go

@@ -0,0 +1,25 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !appengine,!appenginevm
+
+package bleve
+
+import "github.com/blevesearch/bleve/index/store/boltdb"
+
+// in normal environments we configure boltdb as the default storage
+func initDisk() {
+	// default kv store
+	Config.DefaultKVStore = boltdb.Name
+}

vendor/github.com/blevesearch/bleve/doc.go

@@ -0,0 +1,38 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/*
+Package bleve is a library for indexing and searching text.
+
+Example Opening New Index, Indexing Data
+
+    message := struct{
+        Id:   "example"
+        From: "marty.schoch@gmail.com",
+        Body: "bleve indexing is easy",
+    }
+
+    mapping := bleve.NewIndexMapping()
+    index, _ := bleve.New("example.bleve", mapping)
+    index.Index(message.Id, message)
+
+Example Opening Existing Index, Searching Data
+
+    index, _ := bleve.Open("example.bleve")
+    query := bleve.NewQueryStringQuery("bleve")
+    searchRequest := bleve.NewSearchRequest(query)
+    searchResult, _ := index.Search(searchRequest)
+
+*/
+package bleve

vendor/github.com/blevesearch/bleve/document/document.go

@@ -0,0 +1,101 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package document
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/blevesearch/bleve/size"
+)
+
+var reflectStaticSizeDocument int
+
+func init() {
+	var d Document
+	reflectStaticSizeDocument = int(reflect.TypeOf(d).Size())
+}
+
+type Document struct {
+	ID              string  `json:"id"`
+	Fields          []Field `json:"fields"`
+	CompositeFields []*CompositeField
+}
+
+func NewDocument(id string) *Document {
+	return &Document{
+		ID:              id,
+		Fields:          make([]Field, 0),
+		CompositeFields: make([]*CompositeField, 0),
+	}
+}
+
+func (d *Document) Size() int {
+	sizeInBytes := reflectStaticSizeDocument + size.SizeOfPtr +
+		len(d.ID)
+
+	for _, entry := range d.Fields {
+		sizeInBytes += entry.Size()
+	}
+
+	for _, entry := range d.CompositeFields {
+		sizeInBytes += entry.Size()
+	}
+
+	return sizeInBytes
+}
+
+func (d *Document) AddField(f Field) *Document {
+	switch f := f.(type) {
+	case *CompositeField:
+		d.CompositeFields = append(d.CompositeFields, f)
+	default:
+		d.Fields = append(d.Fields, f)
+	}
+	return d
+}
+
+func (d *Document) GoString() string {
+	fields := ""
+	for i, field := range d.Fields {
+		if i != 0 {
+			fields += ", "
+		}
+		fields += fmt.Sprintf("%#v", field)
+	}
+	compositeFields := ""
+	for i, field := range d.CompositeFields {
+		if i != 0 {
+			compositeFields += ", "
+		}
+		compositeFields += fmt.Sprintf("%#v", field)
+	}
+	return fmt.Sprintf("&document.Document{ID:%s, Fields: %s, CompositeFields: %s}", d.ID, fields, compositeFields)
+}
+
+func (d *Document) NumPlainTextBytes() uint64 {
+	rv := uint64(0)
+	for _, field := range d.Fields {
+		rv += field.NumPlainTextBytes()
+	}
+	for _, compositeField := range d.CompositeFields {
+		for _, field := range d.Fields {
+			if compositeField.includesField(field.Name()) {
+				rv += field.NumPlainTextBytes()
+			}
+		}
+	}
+	return rv
+}

vendor/github.com/blevesearch/bleve/document/field.go

@@ -0,0 +1,41 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package document
+
+import (
+	"github.com/blevesearch/bleve/analysis"
+)
+
+type Field interface {
+	// Name returns the path of the field from the root DocumentMapping.
+	// A root field path is "field", a subdocument field is "parent.field".
+	Name() string
+	// ArrayPositions returns the intermediate document and field indices
+	// required to resolve the field value in the document. For example, if the
+	// field path is "doc1.doc2.field" where doc1 and doc2 are slices or
+	// arrays, ArrayPositions returns 2 indices used to resolve "doc2" value in
+	// "doc1", then "field" in "doc2".
+	ArrayPositions() []uint64
+	Options() IndexingOptions
+	Analyze() (int, analysis.TokenFrequencies)
+	Value() []byte
+
+	// NumPlainTextBytes should return the number of plain text bytes
+	// that this field represents - this is a common metric for tracking
+	// the rate of indexing
+	NumPlainTextBytes() uint64
+
+	Size() int
+}

vendor/github.com/blevesearch/bleve/document/field_boolean.go

@@ -0,0 +1,123 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package document
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/blevesearch/bleve/analysis"
+	"github.com/blevesearch/bleve/size"
+)
+
+var reflectStaticSizeBooleanField int
+
+func init() {
+	var f BooleanField
+	reflectStaticSizeBooleanField = int(reflect.TypeOf(f).Size())
+}
+
+const DefaultBooleanIndexingOptions = StoreField | IndexField | DocValues
+
+type BooleanField struct {
+	name              string
+	arrayPositions    []uint64
+	options           IndexingOptions
+	value             []byte
+	numPlainTextBytes uint64
+}
+
+func (b *BooleanField) Size() int {
+	return reflectStaticSizeBooleanField + size.SizeOfPtr +
+		len(b.name) +
+		len(b.arrayPositions)*size.SizeOfUint64 +
+		len(b.value)
+}
+
+func (b *BooleanField) Name() string {
+	return b.name
+}
+
+func (b *BooleanField) ArrayPositions() []uint64 {
+	return b.arrayPositions
+}
+
+func (b *BooleanField) Options() IndexingOptions {
+	return b.options
+}
+
+func (b *BooleanField) Analyze() (int, analysis.TokenFrequencies) {
+	tokens := make(analysis.TokenStream, 0)
+	tokens = append(tokens, &analysis.Token{
+		Start:    0,
+		End:      len(b.value),
+		Term:     b.value,
+		Position: 1,
+		Type:     analysis.Boolean,
+	})
+
+	fieldLength := len(tokens)
+	tokenFreqs := analysis.TokenFrequency(tokens, b.arrayPositions, b.options.IncludeTermVectors())
+	return fieldLength, tokenFreqs
+}
+
+func (b *BooleanField) Value() []byte {
+	return b.value
+}
+
+func (b *BooleanField) Boolean() (bool, error) {
+	if len(b.value) == 1 {
+		return b.value[0] == 'T', nil
+	}
+	return false, fmt.Errorf("boolean field has %d bytes", len(b.value))
+}
+
+func (b *BooleanField) GoString() string {
+	return fmt.Sprintf("&document.BooleanField{Name:%s, Options: %s, Value: %s}", b.name, b.options, b.value)
+}
+
+func (b *BooleanField) NumPlainTextBytes() uint64 {
+	return b.numPlainTextBytes
+}
+
+func NewBooleanFieldFromBytes(name string, arrayPositions []uint64, value []byte) *BooleanField {
+	return &BooleanField{
+		name:              name,
+		arrayPositions:    arrayPositions,
+		value:             value,
+		options:           DefaultNumericIndexingOptions,
+		numPlainTextBytes: uint64(len(value)),
+	}
+}
+
+func NewBooleanField(name string, arrayPositions []uint64, b bool) *BooleanField {
+	return NewBooleanFieldWithIndexingOptions(name, arrayPositions, b, DefaultNumericIndexingOptions)
+}
+
+func NewBooleanFieldWithIndexingOptions(name string, arrayPositions []uint64, b bool, options IndexingOptions) *BooleanField {
+	numPlainTextBytes := 5
+	v := []byte("F")
+	if b {
+		numPlainTextBytes = 4
+		v = []byte("T")
+	}
+	return &BooleanField{
+		name:              name,
+		arrayPositions:    arrayPositions,
+		value:             v,
+		options:           options,
+		numPlainTextBytes: uint64(numPlainTextBytes),
+	}
+}

vendor/github.com/blevesearch/bleve/document/field_composite.go

@@ -0,0 +1,124 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package document
+
+import (
+	"reflect"
+
+	"github.com/blevesearch/bleve/analysis"
+	"github.com/blevesearch/bleve/size"
+)
+
+var reflectStaticSizeCompositeField int
+
+func init() {
+	var cf CompositeField
+	reflectStaticSizeCompositeField = int(reflect.TypeOf(cf).Size())
+}
+
+const DefaultCompositeIndexingOptions = IndexField
+
+type CompositeField struct {
+	name                 string
+	includedFields       map[string]bool
+	excludedFields       map[string]bool
+	defaultInclude       bool
+	options              IndexingOptions
+	totalLength          int
+	compositeFrequencies analysis.TokenFrequencies
+}
+
+func NewCompositeField(name string, defaultInclude bool, include []string, exclude []string) *CompositeField {
+	return NewCompositeFieldWithIndexingOptions(name, defaultInclude, include, exclude, DefaultCompositeIndexingOptions)
+}
+
+func NewCompositeFieldWithIndexingOptions(name string, defaultInclude bool, include []string, exclude []string, options IndexingOptions) *CompositeField {
+	rv := &CompositeField{
+		name:                 name,
+		options:              options,
+		defaultInclude:       defaultInclude,
+		includedFields:       make(map[string]bool, len(include)),
+		excludedFields:       make(map[string]bool, len(exclude)),
+		compositeFrequencies: make(analysis.TokenFrequencies),
+	}
+
+	for _, i := range include {
+		rv.includedFields[i] = true
+	}
+	for _, e := range exclude {
+		rv.excludedFields[e] = true
+	}
+
+	return rv
+}
+
+func (c *CompositeField) Size() int {
+	sizeInBytes := reflectStaticSizeCompositeField + size.SizeOfPtr +
+		len(c.name)
+
+	for k, _ := range c.includedFields {
+		sizeInBytes += size.SizeOfString + len(k) + size.SizeOfBool
+	}
+
+	for k, _ := range c.excludedFields {
+		sizeInBytes += size.SizeOfString + len(k) + size.SizeOfBool
+	}
+
+	return sizeInBytes
+}
+
+func (c *CompositeField) Name() string {
+	return c.name
+}
+
+func (c *CompositeField) ArrayPositions() []uint64 {
+	return []uint64{}
+}
+
+func (c *CompositeField) Options() IndexingOptions {
+	return c.options
+}
+
+func (c *CompositeField) Analyze() (int, analysis.TokenFrequencies) {
+	return c.totalLength, c.compositeFrequencies
+}
+
+func (c *CompositeField) Value() []byte {
+	return []byte{}
+}
+
+func (c *CompositeField) NumPlainTextBytes() uint64 {
+	return 0
+}
+
+func (c *CompositeField) includesField(field string) bool {
+	shouldInclude := c.defaultInclude
+	_, fieldShouldBeIncluded := c.includedFields[field]
+	if fieldShouldBeIncluded {
+		shouldInclude = true
+	}
+	_, fieldShouldBeExcluded := c.excludedFields[field]
+	if fieldShouldBeExcluded {
+		shouldInclude = false
+	}
+	return shouldInclude
+}
+
+func (c *CompositeField) Compose(field string, length int, freq analysis.TokenFrequencies) {
+	if c.includesField(field) {
+		c.totalLength += length
+		c.compositeFrequencies.MergeAll(field, freq)
+	}
+}

vendor/github.com/blevesearch/bleve/document/field_datetime.go

@@ -0,0 +1,159 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package document
+
+import (
+	"fmt"
+	"math"
+	"reflect"
+	"time"
+
+	"github.com/blevesearch/bleve/analysis"
+	"github.com/blevesearch/bleve/numeric"
+	"github.com/blevesearch/bleve/size"
+)
+
+var reflectStaticSizeDateTimeField int
+
+func init() {
+	var f DateTimeField
+	reflectStaticSizeDateTimeField = int(reflect.TypeOf(f).Size())
+}
+
+const DefaultDateTimeIndexingOptions = StoreField | IndexField | DocValues
+const DefaultDateTimePrecisionStep uint = 4
+
+var MinTimeRepresentable = time.Unix(0, math.MinInt64)
+var MaxTimeRepresentable = time.Unix(0, math.MaxInt64)
+
+type DateTimeField struct {
+	name              string
+	arrayPositions    []uint64
+	options           IndexingOptions
+	value             numeric.PrefixCoded
+	numPlainTextBytes uint64
+}
+
+func (n *DateTimeField) Size() int {
+	return reflectStaticSizeDateTimeField + size.SizeOfPtr +
+		len(n.name) +
+		len(n.arrayPositions)*size.SizeOfUint64
+}
+
+func (n *DateTimeField) Name() string {
+	return n.name
+}
+
+func (n *DateTimeField) ArrayPositions() []uint64 {
+	return n.arrayPositions
+}
+
+func (n *DateTimeField) Options() IndexingOptions {
+	return n.options
+}
+
+func (n *DateTimeField) Analyze() (int, analysis.TokenFrequencies) {
+	tokens := make(analysis.TokenStream, 0)
+	tokens = append(tokens, &analysis.Token{
+		Start:    0,
+		End:      len(n.value),
+		Term:     n.value,
+		Position: 1,
+		Type:     analysis.DateTime,
+	})
+
+	original, err := n.value.Int64()
+	if err == nil {
+
+		shift := DefaultDateTimePrecisionStep
+		for shift < 64 {
+			shiftEncoded, err := numeric.NewPrefixCodedInt64(original, shift)
+			if err != nil {
+				break
+			}
+			token := analysis.Token{
+				Start:    0,
+				End:      len(shiftEncoded),
+				Term:     shiftEncoded,
+				Position: 1,
+				Type:     analysis.DateTime,
+			}
+			tokens = append(tokens, &token)
+			shift += DefaultDateTimePrecisionStep
+		}
+	}
+
+	fieldLength := len(tokens)
+	tokenFreqs := analysis.TokenFrequency(tokens, n.arrayPositions, n.options.IncludeTermVectors())
+	return fieldLength, tokenFreqs
+}
+
+func (n *DateTimeField) Value() []byte {
+	return n.value
+}
+
+func (n *DateTimeField) DateTime() (time.Time, error) {
+	i64, err := n.value.Int64()
+	if err != nil {
+		return time.Time{}, err
+	}
+	return time.Unix(0, i64).UTC(), nil
+}
+
+func (n *DateTimeField) GoString() string {
+	return fmt.Sprintf("&document.DateField{Name:%s, Options: %s, Value: %s}", n.name, n.options, n.value)
+}
+
+func (n *DateTimeField) NumPlainTextBytes() uint64 {
+	return n.numPlainTextBytes
+}
+
+func NewDateTimeFieldFromBytes(name string, arrayPositions []uint64, value []byte) *DateTimeField {
+	return &DateTimeField{
+		name:              name,
+		arrayPositions:    arrayPositions,
+		value:             value,
+		options:           DefaultDateTimeIndexingOptions,
+		numPlainTextBytes: uint64(len(value)),
+	}
+}
+
+func NewDateTimeField(name string, arrayPositions []uint64, dt time.Time) (*DateTimeField, error) {
+	return NewDateTimeFieldWithIndexingOptions(name, arrayPositions, dt, DefaultDateTimeIndexingOptions)
+}
+
+func NewDateTimeFieldWithIndexingOptions(name string, arrayPositions []uint64, dt time.Time, options IndexingOptions) (*DateTimeField, error) {
+	if canRepresent(dt) {
+		dtInt64 := dt.UnixNano()
+		prefixCoded := numeric.MustNewPrefixCodedInt64(dtInt64, 0)
+		return &DateTimeField{
+			name:           name,
+			arrayPositions: arrayPositions,
+			value:          prefixCoded,
+			options:        options,
+			// not correct, just a place holder until we revisit how fields are
+			// represented and can fix this better
+			numPlainTextBytes: uint64(8),
+		}, nil
+	}
+	return nil, fmt.Errorf("cannot represent %s in this type", dt)
+}
+
+func canRepresent(dt time.Time) bool {
+	if dt.Before(MinTimeRepresentable) || dt.After(MaxTimeRepresentable) {
+		return false
+	}
+	return true
+}

vendor/github.com/blevesearch/bleve/document/field_geopoint.go

@@ -0,0 +1,152 @@
+//  Copyright (c) 2017 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package document
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/blevesearch/bleve/analysis"
+	"github.com/blevesearch/bleve/geo"
+	"github.com/blevesearch/bleve/numeric"
+	"github.com/blevesearch/bleve/size"
+)
+
+var reflectStaticSizeGeoPointField int
+
+func init() {
+	var f GeoPointField
+	reflectStaticSizeGeoPointField = int(reflect.TypeOf(f).Size())
+}
+
+var GeoPrecisionStep uint = 9
+
+type GeoPointField struct {
+	name              string
+	arrayPositions    []uint64
+	options           IndexingOptions
+	value             numeric.PrefixCoded
+	numPlainTextBytes uint64
+}
+
+func (n *GeoPointField) Size() int {
+	return reflectStaticSizeGeoPointField + size.SizeOfPtr +
+		len(n.name) +
+		len(n.arrayPositions)*size.SizeOfUint64
+}
+
+func (n *GeoPointField) Name() string {
+	return n.name
+}
+
+func (n *GeoPointField) ArrayPositions() []uint64 {
+	return n.arrayPositions
+}
+
+func (n *GeoPointField) Options() IndexingOptions {
+	return n.options
+}
+
+func (n *GeoPointField) Analyze() (int, analysis.TokenFrequencies) {
+	tokens := make(analysis.TokenStream, 0)
+	tokens = append(tokens, &analysis.Token{
+		Start:    0,
+		End:      len(n.value),
+		Term:     n.value,
+		Position: 1,
+		Type:     analysis.Numeric,
+	})
+
+	original, err := n.value.Int64()
+	if err == nil {
+
+		shift := GeoPrecisionStep
+		for shift < 64 {
+			shiftEncoded, err := numeric.NewPrefixCodedInt64(original, shift)
+			if err != nil {
+				break
+			}
+			token := analysis.Token{
+				Start:    0,
+				End:      len(shiftEncoded),
+				Term:     shiftEncoded,
+				Position: 1,
+				Type:     analysis.Numeric,
+			}
+			tokens = append(tokens, &token)
+			shift += GeoPrecisionStep
+		}
+	}
+
+	fieldLength := len(tokens)
+	tokenFreqs := analysis.TokenFrequency(tokens, n.arrayPositions, n.options.IncludeTermVectors())
+	return fieldLength, tokenFreqs
+}
+
+func (n *GeoPointField) Value() []byte {
+	return n.value
+}
+
+func (n *GeoPointField) Lon() (float64, error) {
+	i64, err := n.value.Int64()
+	if err != nil {
+		return 0.0, err
+	}
+	return geo.MortonUnhashLon(uint64(i64)), nil
+}
+
+func (n *GeoPointField) Lat() (float64, error) {
+	i64, err := n.value.Int64()
+	if err != nil {
+		return 0.0, err
+	}
+	return geo.MortonUnhashLat(uint64(i64)), nil
+}
+
+func (n *GeoPointField) GoString() string {
+	return fmt.Sprintf("&document.GeoPointField{Name:%s, Options: %s, Value: %s}", n.name, n.options, n.value)
+}
+
+func (n *GeoPointField) NumPlainTextBytes() uint64 {
+	return n.numPlainTextBytes
+}
+
+func NewGeoPointFieldFromBytes(name string, arrayPositions []uint64, value []byte) *GeoPointField {
+	return &GeoPointField{
+		name:              name,
+		arrayPositions:    arrayPositions,
+		value:             value,
+		options:           DefaultNumericIndexingOptions,
+		numPlainTextBytes: uint64(len(value)),
+	}
+}
+
+func NewGeoPointField(name string, arrayPositions []uint64, lon, lat float64) *GeoPointField {
+	return NewGeoPointFieldWithIndexingOptions(name, arrayPositions, lon, lat, DefaultNumericIndexingOptions)
+}
+
+func NewGeoPointFieldWithIndexingOptions(name string, arrayPositions []uint64, lon, lat float64, options IndexingOptions) *GeoPointField {
+	mhash := geo.MortonHash(lon, lat)
+	prefixCoded := numeric.MustNewPrefixCodedInt64(int64(mhash), 0)
+	return &GeoPointField{
+		name:           name,
+		arrayPositions: arrayPositions,
+		value:          prefixCoded,
+		options:        options,
+		// not correct, just a place holder until we revisit how fields are
+		// represented and can fix this better
+		numPlainTextBytes: uint64(8),
+	}
+}

vendor/github.com/blevesearch/bleve/document/field_numeric.go

@@ -0,0 +1,145 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package document
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/blevesearch/bleve/analysis"
+	"github.com/blevesearch/bleve/numeric"
+	"github.com/blevesearch/bleve/size"
+)
+
+var reflectStaticSizeNumericField int
+
+func init() {
+	var f NumericField
+	reflectStaticSizeNumericField = int(reflect.TypeOf(f).Size())
+}
+
+const DefaultNumericIndexingOptions = StoreField | IndexField | DocValues
+
+const DefaultPrecisionStep uint = 4
+
+type NumericField struct {
+	name              string
+	arrayPositions    []uint64
+	options           IndexingOptions
+	value             numeric.PrefixCoded
+	numPlainTextBytes uint64
+}
+
+func (n *NumericField) Size() int {
+	return reflectStaticSizeNumericField + size.SizeOfPtr +
+		len(n.name) +
+		len(n.arrayPositions)*size.SizeOfPtr
+}
+
+func (n *NumericField) Name() string {
+	return n.name
+}
+
+func (n *NumericField) ArrayPositions() []uint64 {
+	return n.arrayPositions
+}
+
+func (n *NumericField) Options() IndexingOptions {
+	return n.options
+}
+
+func (n *NumericField) Analyze() (int, analysis.TokenFrequencies) {
+	tokens := make(analysis.TokenStream, 0)
+	tokens = append(tokens, &analysis.Token{
+		Start:    0,
+		End:      len(n.value),
+		Term:     n.value,
+		Position: 1,
+		Type:     analysis.Numeric,
+	})
+
+	original, err := n.value.Int64()
+	if err == nil {
+
+		shift := DefaultPrecisionStep
+		for shift < 64 {
+			shiftEncoded, err := numeric.NewPrefixCodedInt64(original, shift)
+			if err != nil {
+				break
+			}
+			token := analysis.Token{
+				Start:    0,
+				End:      len(shiftEncoded),
+				Term:     shiftEncoded,
+				Position: 1,
+				Type:     analysis.Numeric,
+			}
+			tokens = append(tokens, &token)
+			shift += DefaultPrecisionStep
+		}
+	}
+
+	fieldLength := len(tokens)
+	tokenFreqs := analysis.TokenFrequency(tokens, n.arrayPositions, n.options.IncludeTermVectors())
+	return fieldLength, tokenFreqs
+}
+
+func (n *NumericField) Value() []byte {
+	return n.value
+}
+
+func (n *NumericField) Number() (float64, error) {
+	i64, err := n.value.Int64()
+	if err != nil {
+		return 0.0, err
+	}
+	return numeric.Int64ToFloat64(i64), nil
+}
+
+func (n *NumericField) GoString() string {
+	return fmt.Sprintf("&document.NumericField{Name:%s, Options: %s, Value: %s}", n.name, n.options, n.value)
+}
+
+func (n *NumericField) NumPlainTextBytes() uint64 {
+	return n.numPlainTextBytes
+}
+
+func NewNumericFieldFromBytes(name string, arrayPositions []uint64, value []byte) *NumericField {
+	return &NumericField{
+		name:              name,
+		arrayPositions:    arrayPositions,
+		value:             value,
+		options:           DefaultNumericIndexingOptions,
+		numPlainTextBytes: uint64(len(value)),
+	}
+}
+
+func NewNumericField(name string, arrayPositions []uint64, number float64) *NumericField {
+	return NewNumericFieldWithIndexingOptions(name, arrayPositions, number, DefaultNumericIndexingOptions)
+}
+
+func NewNumericFieldWithIndexingOptions(name string, arrayPositions []uint64, number float64, options IndexingOptions) *NumericField {
+	numberInt64 := numeric.Float64ToInt64(number)
+	prefixCoded := numeric.MustNewPrefixCodedInt64(numberInt64, 0)
+	return &NumericField{
+		name:           name,
+		arrayPositions: arrayPositions,
+		value:          prefixCoded,
+		options:        options,
+		// not correct, just a place holder until we revisit how fields are
+		// represented and can fix this better
+		numPlainTextBytes: uint64(8),
+	}
+}

vendor/github.com/blevesearch/bleve/document/field_text.go

@@ -0,0 +1,139 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package document
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/blevesearch/bleve/analysis"
+	"github.com/blevesearch/bleve/size"
+)
+
+var reflectStaticSizeTextField int
+
+func init() {
+	var f TextField
+	reflectStaticSizeTextField = int(reflect.TypeOf(f).Size())
+}
+
+const DefaultTextIndexingOptions = IndexField | DocValues
+
+type TextField struct {
+	name              string
+	arrayPositions    []uint64
+	options           IndexingOptions
+	analyzer          *analysis.Analyzer
+	value             []byte
+	numPlainTextBytes uint64
+}
+
+func (t *TextField) Size() int {
+	return reflectStaticSizeTextField + size.SizeOfPtr +
+		len(t.name) +
+		len(t.arrayPositions)*size.SizeOfUint64 +
+		len(t.value)
+}
+
+func (t *TextField) Name() string {
+	return t.name
+}
+
+func (t *TextField) ArrayPositions() []uint64 {
+	return t.arrayPositions
+}
+
+func (t *TextField) Options() IndexingOptions {
+	return t.options
+}
+
+func (t *TextField) Analyze() (int, analysis.TokenFrequencies) {
+	var tokens analysis.TokenStream
+	if t.analyzer != nil {
+		bytesToAnalyze := t.Value()
+		if t.options.IsStored() {
+			// need to copy
+			bytesCopied := make([]byte, len(bytesToAnalyze))
+			copy(bytesCopied, bytesToAnalyze)
+			bytesToAnalyze = bytesCopied
+		}
+		tokens = t.analyzer.Analyze(bytesToAnalyze)
+	} else {
+		tokens = analysis.TokenStream{
+			&analysis.Token{
+				Start:    0,
+				End:      len(t.value),
+				Term:     t.value,
+				Position: 1,
+				Type:     analysis.AlphaNumeric,
+			},
+		}
+	}
+	fieldLength := len(tokens) // number of tokens in this doc field
+	tokenFreqs := analysis.TokenFrequency(tokens, t.arrayPositions, t.options.IncludeTermVectors())
+	return fieldLength, tokenFreqs
+}
+
+func (t *TextField) Analyzer() *analysis.Analyzer {
+	return t.analyzer
+}
+
+func (t *TextField) Value() []byte {
+	return t.value
+}
+
+func (t *TextField) GoString() string {
+	return fmt.Sprintf("&document.TextField{Name:%s, Options: %s, Analyzer: %v, Value: %s, ArrayPositions: %v}", t.name, t.options, t.analyzer, t.value, t.arrayPositions)
+}
+
+func (t *TextField) NumPlainTextBytes() uint64 {
+	return t.numPlainTextBytes
+}
+
+func NewTextField(name string, arrayPositions []uint64, value []byte) *TextField {
+	return NewTextFieldWithIndexingOptions(name, arrayPositions, value, DefaultTextIndexingOptions)
+}
+
+func NewTextFieldWithIndexingOptions(name string, arrayPositions []uint64, value []byte, options IndexingOptions) *TextField {
+	return &TextField{
+		name:              name,
+		arrayPositions:    arrayPositions,
+		options:           options,
+		value:             value,
+		numPlainTextBytes: uint64(len(value)),
+	}
+}
+
+func NewTextFieldWithAnalyzer(name string, arrayPositions []uint64, value []byte, analyzer *analysis.Analyzer) *TextField {
+	return &TextField{
+		name:              name,
+		arrayPositions:    arrayPositions,
+		options:           DefaultTextIndexingOptions,
+		analyzer:          analyzer,
+		value:             value,
+		numPlainTextBytes: uint64(len(value)),
+	}
+}
+
+func NewTextFieldCustom(name string, arrayPositions []uint64, value []byte, options IndexingOptions, analyzer *analysis.Analyzer) *TextField {
+	return &TextField{
+		name:              name,
+		arrayPositions:    arrayPositions,
+		options:           options,
+		analyzer:          analyzer,
+		value:             value,
+		numPlainTextBytes: uint64(len(value)),
+	}
+}

vendor/github.com/blevesearch/bleve/document/indexing_options.go

@@ -0,0 +1,66 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package document
+
+type IndexingOptions int
+
+const (
+	IndexField IndexingOptions = 1 << iota
+	StoreField
+	IncludeTermVectors
+	DocValues
+)
+
+func (o IndexingOptions) IsIndexed() bool {
+	return o&IndexField != 0
+}
+
+func (o IndexingOptions) IsStored() bool {
+	return o&StoreField != 0
+}
+
+func (o IndexingOptions) IncludeTermVectors() bool {
+	return o&IncludeTermVectors != 0
+}
+
+func (o IndexingOptions) IncludeDocValues() bool {
+	return o&DocValues != 0
+}
+
+func (o IndexingOptions) String() string {
+	rv := ""
+	if o.IsIndexed() {
+		rv += "INDEXED"
+	}
+	if o.IsStored() {
+		if rv != "" {
+			rv += ", "
+		}
+		rv += "STORE"
+	}
+	if o.IncludeTermVectors() {
+		if rv != "" {
+			rv += ", "
+		}
+		rv += "TV"
+	}
+	if o.IncludeDocValues() {
+		if rv != "" {
+			rv += ", "
+		}
+		rv += "DV"
+	}
+	return rv
+}

vendor/github.com/blevesearch/bleve/error.go

@@ -0,0 +1,52 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bleve
+
+// Constant Error values which can be compared to determine the type of error
+const (
+	ErrorIndexPathExists Error = iota
+	ErrorIndexPathDoesNotExist
+	ErrorIndexMetaMissing
+	ErrorIndexMetaCorrupt
+	ErrorUnknownStorageType
+	ErrorIndexClosed
+	ErrorAliasMulti
+	ErrorAliasEmpty
+	ErrorUnknownIndexType
+	ErrorEmptyID
+	ErrorIndexReadInconsistency
+)
+
+// Error represents a more strongly typed bleve error for detecting
+// and handling specific types of errors.
+type Error int
+
+func (e Error) Error() string {
+	return errorMessages[e]
+}
+
+var errorMessages = map[Error]string{
+	ErrorIndexPathExists:        "cannot create new index, path already exists",
+	ErrorIndexPathDoesNotExist:  "cannot open index, path does not exist",
+	ErrorIndexMetaMissing:       "cannot open index, metadata missing",
+	ErrorIndexMetaCorrupt:       "cannot open index, metadata corrupt",
+	ErrorUnknownStorageType:     "unknown storage type",
+	ErrorIndexClosed:            "index is closed",
+	ErrorAliasMulti:             "cannot perform single index operation on multiple index alias",
+	ErrorAliasEmpty:             "cannot perform operation on empty alias",
+	ErrorUnknownIndexType:       "unknown index type",
+	ErrorEmptyID:                "document ID cannot be empty",
+	ErrorIndexReadInconsistency: "index read inconsistency detected",
+}

vendor/github.com/blevesearch/bleve/geo/README.md

@@ -0,0 +1,9 @@
+# geo support in bleve
+
+First, all of this geo code is a Go adaptation of the [Lucene 5.3.2 sandbox geo support](https://lucene.apache.org/core/5_3_2/sandbox/org/apache/lucene/util/package-summary.html).
+
+## Notes
+
+- All of the APIs will use float64 for lon/lat values.
+- When describing a point in function arguments or return values, we always use the order lon, lat.
+- High level APIs will use TopLeft and BottomRight to describe bounding boxes.  This may not map cleanly to min/max lon/lat when crossing the dateline.  The lower level APIs will use min/max lon/lat and require the higher-level code to split boxes accordingly.

vendor/github.com/blevesearch/bleve/geo/geo.go

@@ -0,0 +1,210 @@
+//  Copyright (c) 2017 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package geo
+
+import (
+	"fmt"
+	"math"
+
+	"github.com/blevesearch/bleve/numeric"
+)
+
+// GeoBits is the number of bits used for a single geo point
+// Currently this is 32bits for lon and 32bits for lat
+var GeoBits uint = 32
+
+var minLon = -180.0
+var minLat = -90.0
+var maxLon = 180.0
+var maxLat = 90.0
+var minLonRad = minLon * degreesToRadian
+var minLatRad = minLat * degreesToRadian
+var maxLonRad = maxLon * degreesToRadian
+var maxLatRad = maxLat * degreesToRadian
+var geoTolerance = 1e-6
+var lonScale = float64((uint64(0x1)<<GeoBits)-1) / 360.0
+var latScale = float64((uint64(0x1)<<GeoBits)-1) / 180.0
+
+var geoHashMaxLength = 12
+
+// Point represents a geo point.
+type Point struct {
+	Lon float64 `json:"lon"`
+	Lat float64 `json:"lat"`
+}
+
+// MortonHash computes the morton hash value for the provided geo point
+// This point is ordered as lon, lat.
+func MortonHash(lon, lat float64) uint64 {
+	return numeric.Interleave(scaleLon(lon), scaleLat(lat))
+}
+
+func scaleLon(lon float64) uint64 {
+	rv := uint64((lon - minLon) * lonScale)
+	return rv
+}
+
+func scaleLat(lat float64) uint64 {
+	rv := uint64((lat - minLat) * latScale)
+	return rv
+}
+
+// MortonUnhashLon extracts the longitude value from the provided morton hash.
+func MortonUnhashLon(hash uint64) float64 {
+	return unscaleLon(numeric.Deinterleave(hash))
+}
+
+// MortonUnhashLat extracts the latitude value from the provided morton hash.
+func MortonUnhashLat(hash uint64) float64 {
+	return unscaleLat(numeric.Deinterleave(hash >> 1))
+}
+
+func unscaleLon(lon uint64) float64 {
+	return (float64(lon) / lonScale) + minLon
+}
+
+func unscaleLat(lat uint64) float64 {
+	return (float64(lat) / latScale) + minLat
+}
+
+// compareGeo will compare two float values and see if they are the same
+// taking into consideration a known geo tolerance.
+func compareGeo(a, b float64) float64 {
+	compare := a - b
+	if math.Abs(compare) <= geoTolerance {
+		return 0
+	}
+	return compare
+}
+
+// RectIntersects checks whether rectangles a and b intersect
+func RectIntersects(aMinX, aMinY, aMaxX, aMaxY, bMinX, bMinY, bMaxX, bMaxY float64) bool {
+	return !(aMaxX < bMinX || aMinX > bMaxX || aMaxY < bMinY || aMinY > bMaxY)
+}
+
+// RectWithin checks whether box a is within box b
+func RectWithin(aMinX, aMinY, aMaxX, aMaxY, bMinX, bMinY, bMaxX, bMaxY float64) bool {
+	rv := !(aMinX < bMinX || aMinY < bMinY || aMaxX > bMaxX || aMaxY > bMaxY)
+	return rv
+}
+
+// BoundingBoxContains checks whether the lon/lat point is within the box
+func BoundingBoxContains(lon, lat, minLon, minLat, maxLon, maxLat float64) bool {
+	return compareGeo(lon, minLon) >= 0 && compareGeo(lon, maxLon) <= 0 &&
+		compareGeo(lat, minLat) >= 0 && compareGeo(lat, maxLat) <= 0
+}
+
+const degreesToRadian = math.Pi / 180
+const radiansToDegrees = 180 / math.Pi
+
+// DegreesToRadians converts an angle in degrees to radians
+func DegreesToRadians(d float64) float64 {
+	return d * degreesToRadian
+}
+
+// RadiansToDegrees converts an angle in radians to degress
+func RadiansToDegrees(r float64) float64 {
+	return r * radiansToDegrees
+}
+
+var earthMeanRadiusMeters = 6371008.7714
+
+func RectFromPointDistance(lon, lat, dist float64) (float64, float64, float64, float64, error) {
+	err := checkLongitude(lon)
+	if err != nil {
+		return 0, 0, 0, 0, err
+	}
+	err = checkLatitude(lat)
+	if err != nil {
+		return 0, 0, 0, 0, err
+	}
+	radLon := DegreesToRadians(lon)
+	radLat := DegreesToRadians(lat)
+	radDistance := (dist + 7e-2) / earthMeanRadiusMeters
+
+	minLatL := radLat - radDistance
+	maxLatL := radLat + radDistance
+
+	var minLonL, maxLonL float64
+	if minLatL > minLatRad && maxLatL < maxLatRad {
+		deltaLon := asin(sin(radDistance) / cos(radLat))
+		minLonL = radLon - deltaLon
+		if minLonL < minLonRad {
+			minLonL += 2 * math.Pi
+		}
+		maxLonL = radLon + deltaLon
+		if maxLonL > maxLonRad {
+			maxLonL -= 2 * math.Pi
+		}
+	} else {
+		// pole is inside distance
+		minLatL = math.Max(minLatL, minLatRad)
+		maxLatL = math.Min(maxLatL, maxLatRad)
+		minLonL = minLonRad
+		maxLonL = maxLonRad
+	}
+
+	return RadiansToDegrees(minLonL),
+		RadiansToDegrees(maxLatL),
+		RadiansToDegrees(maxLonL),
+		RadiansToDegrees(minLatL),
+		nil
+}
+
+func checkLatitude(latitude float64) error {
+	if math.IsNaN(latitude) || latitude < minLat || latitude > maxLat {
+		return fmt.Errorf("invalid latitude %f; must be between %f and %f", latitude, minLat, maxLat)
+	}
+	return nil
+}
+
+func checkLongitude(longitude float64) error {
+	if math.IsNaN(longitude) || longitude < minLon || longitude > maxLon {
+		return fmt.Errorf("invalid longitude %f; must be between %f and %f", longitude, minLon, maxLon)
+	}
+	return nil
+}
+
+func BoundingRectangleForPolygon(polygon []Point) (
+	float64, float64, float64, float64, error) {
+	err := checkLongitude(polygon[0].Lon)
+	if err != nil {
+		return 0, 0, 0, 0, err
+	}
+	err = checkLatitude(polygon[0].Lat)
+	if err != nil {
+		return 0, 0, 0, 0, err
+	}
+	maxY, minY := polygon[0].Lat, polygon[0].Lat
+	maxX, minX := polygon[0].Lon, polygon[0].Lon
+	for i := 1; i < len(polygon); i++ {
+		err := checkLongitude(polygon[i].Lon)
+		if err != nil {
+			return 0, 0, 0, 0, err
+		}
+		err = checkLatitude(polygon[i].Lat)
+		if err != nil {
+			return 0, 0, 0, 0, err
+		}
+
+		maxY = math.Max(maxY, polygon[i].Lat)
+		minY = math.Min(minY, polygon[i].Lat)
+
+		maxX = math.Max(maxX, polygon[i].Lon)
+		minX = math.Min(minX, polygon[i].Lon)
+	}
+
+	return minX, maxY, maxX, minY, nil
+}

vendor/github.com/blevesearch/bleve/geo/geo_dist.go

@@ -0,0 +1,98 @@
+//  Copyright (c) 2017 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package geo
+
+import (
+	"fmt"
+	"math"
+	"strconv"
+	"strings"
+)
+
+type distanceUnit struct {
+	conv     float64
+	suffixes []string
+}
+
+var inch = distanceUnit{0.0254, []string{"in", "inch"}}
+var yard = distanceUnit{0.9144, []string{"yd", "yards"}}
+var feet = distanceUnit{0.3048, []string{"ft", "feet"}}
+var kilom = distanceUnit{1000, []string{"km", "kilometers"}}
+var nauticalm = distanceUnit{1852.0, []string{"nm", "nauticalmiles"}}
+var millim = distanceUnit{0.001, []string{"mm", "millimeters"}}
+var centim = distanceUnit{0.01, []string{"cm", "centimeters"}}
+var miles = distanceUnit{1609.344, []string{"mi", "miles"}}
+var meters = distanceUnit{1, []string{"m", "meters"}}
+
+var distanceUnits = []*distanceUnit{
+	&inch, &yard, &feet, &kilom, &nauticalm, &millim, &centim, &miles, &meters,
+}
+
+// ParseDistance attempts to parse a distance string and return distance in
+// meters.  Example formats supported:
+// "5in" "5inch" "7yd" "7yards" "9ft" "9feet" "11km" "11kilometers"
+// "3nm" "3nauticalmiles" "13mm" "13millimeters" "15cm" "15centimeters"
+// "17mi" "17miles" "19m" "19meters"
+// If the unit cannot be determined, the entire string is parsed and the
+// unit of meters is assumed.
+// If the number portion cannot be parsed, 0 and the parse error are returned.
+func ParseDistance(d string) (float64, error) {
+	for _, unit := range distanceUnits {
+		for _, unitSuffix := range unit.suffixes {
+			if strings.HasSuffix(d, unitSuffix) {
+				parsedNum, err := strconv.ParseFloat(d[0:len(d)-len(unitSuffix)], 64)
+				if err != nil {
+					return 0, err
+				}
+				return parsedNum * unit.conv, nil
+			}
+		}
+	}
+	// no unit matched, try assuming meters?
+	parsedNum, err := strconv.ParseFloat(d, 64)
+	if err != nil {
+		return 0, err
+	}
+	return parsedNum, nil
+}
+
+// ParseDistanceUnit attempts to parse a distance unit and return the
+// multiplier for converting this to meters.  If the unit cannot be parsed
+// then 0 and the error message is returned.
+func ParseDistanceUnit(u string) (float64, error) {
+	for _, unit := range distanceUnits {
+		for _, unitSuffix := range unit.suffixes {
+			if u == unitSuffix {
+				return unit.conv, nil
+			}
+		}
+	}
+	return 0, fmt.Errorf("unknown distance unit: %s", u)
+}
+
+// Haversin computes the distance between two points.
+// This implemenation uses the sloppy math implemenations which trade off
+// accuracy for performance.  The distance returned is in kilometers.
+func Haversin(lon1, lat1, lon2, lat2 float64) float64 {
+	x1 := lat1 * degreesToRadian
+	x2 := lat2 * degreesToRadian
+	h1 := 1 - cos(x1-x2)
+	h2 := 1 - cos((lon1-lon2)*degreesToRadian)
+	h := (h1 + cos(x1)*cos(x2)*h2) / 2
+	avgLat := (x1 + x2) / 2
+	diameter := earthDiameter(avgLat)
+
+	return diameter * asin(math.Min(1, math.Sqrt(h)))
+}

vendor/github.com/blevesearch/bleve/geo/geohash.go

@@ -0,0 +1,111 @@
+//  Copyright (c) 2019 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// This implementation is inspired from the geohash-js
+// ref: https://github.com/davetroy/geohash-js
+
+package geo
+
+// encoding encapsulates an encoding defined by a given base32 alphabet.
+type encoding struct {
+	enc string
+	dec [256]byte
+}
+
+// newEncoding constructs a new encoding defined by the given alphabet,
+// which must be a 32-byte string.
+func newEncoding(encoder string) *encoding {
+	e := new(encoding)
+	e.enc = encoder
+	for i := 0; i < len(e.dec); i++ {
+		e.dec[i] = 0xff
+	}
+	for i := 0; i < len(encoder); i++ {
+		e.dec[encoder[i]] = byte(i)
+	}
+	return e
+}
+
+// base32encoding with the Geohash alphabet.
+var base32encoding = newEncoding("0123456789bcdefghjkmnpqrstuvwxyz")
+
+var masks = []uint64{16, 8, 4, 2, 1}
+
+// DecodeGeoHash decodes the string geohash faster with
+// higher precision. This api is in experimental phase.
+func DecodeGeoHash(geoHash string) (float64, float64) {
+	even := true
+	lat := []float64{-90.0, 90.0}
+	lon := []float64{-180.0, 180.0}
+
+	for i := 0; i < len(geoHash); i++ {
+		cd := uint64(base32encoding.dec[geoHash[i]])
+		for j := 0; j < 5; j++ {
+			if even {
+				if cd&masks[j] > 0 {
+					lon[0] = (lon[0] + lon[1]) / 2
+				} else {
+					lon[1] = (lon[0] + lon[1]) / 2
+				}
+			} else {
+				if cd&masks[j] > 0 {
+					lat[0] = (lat[0] + lat[1]) / 2
+				} else {
+					lat[1] = (lat[0] + lat[1]) / 2
+				}
+			}
+			even = !even
+		}
+	}
+
+	return (lat[0] + lat[1]) / 2, (lon[0] + lon[1]) / 2
+}
+
+func EncodeGeoHash(lat, lon float64) string {
+	even := true
+	lats := []float64{-90.0, 90.0}
+	lons := []float64{-180.0, 180.0}
+	precision := 12
+	var ch, bit uint64
+	var geoHash string
+
+	for len(geoHash) < precision {
+		if even {
+			mid := (lons[0] + lons[1]) / 2
+			if lon > mid {
+				ch |= masks[bit]
+				lons[0] = mid
+			} else {
+				lons[1] = mid
+			}
+		} else {
+			mid := (lats[0] + lats[1]) / 2
+			if lat > mid {
+				ch |= masks[bit]
+				lats[0] = mid
+			} else {
+				lats[1] = mid
+			}
+		}
+		even = !even
+		if bit < 4 {
+			bit++
+		} else {
+			geoHash += string(base32encoding.enc[ch])
+			ch = 0
+			bit = 0
+		}
+	}
+
+	return geoHash
+}

vendor/github.com/blevesearch/bleve/geo/parse.go

@@ -0,0 +1,181 @@
+//  Copyright (c) 2017 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package geo
+
+import (
+	"reflect"
+	"strconv"
+	"strings"
+)
+
+// ExtractGeoPoint takes an arbitrary interface{} and tries it's best to
+// interpret it is as geo point.  Supported formats:
+// Container:
+// slice length 2 (GeoJSON)
+//  first element lon, second element lat
+// string (coordinates separated by comma, or a geohash)
+//  first element lat, second element lon
+// map[string]interface{}
+//  exact keys lat and lon or lng
+// struct
+//  w/exported fields case-insensitive match on lat and lon or lng
+// struct
+//  satisfying Later and Loner or Lnger interfaces
+//
+// in all cases values must be some sort of numeric-like thing: int/uint/float
+func ExtractGeoPoint(thing interface{}) (lon, lat float64, success bool) {
+	var foundLon, foundLat bool
+
+	thingVal := reflect.ValueOf(thing)
+	if !thingVal.IsValid() {
+		return lon, lat, false
+	}
+
+	thingTyp := thingVal.Type()
+
+	// is it a slice
+	if thingVal.Kind() == reflect.Slice {
+		// must be length 2
+		if thingVal.Len() == 2 {
+			first := thingVal.Index(0)
+			if first.CanInterface() {
+				firstVal := first.Interface()
+				lon, foundLon = extractNumericVal(firstVal)
+			}
+			second := thingVal.Index(1)
+			if second.CanInterface() {
+				secondVal := second.Interface()
+				lat, foundLat = extractNumericVal(secondVal)
+			}
+		}
+	}
+
+	// is it a string
+	if thingVal.Kind() == reflect.String {
+		geoStr := thingVal.Interface().(string)
+		if strings.Contains(geoStr, ",") {
+			// geo point with coordinates split by comma
+			points := strings.Split(geoStr, ",")
+			for i, point := range points {
+				// trim any leading or trailing white spaces
+				points[i] = strings.TrimSpace(point)
+			}
+			if len(points) == 2 {
+				var err error
+				lat, err = strconv.ParseFloat(points[0], 64)
+				if err == nil {
+					foundLat = true
+				}
+				lon, err = strconv.ParseFloat(points[1], 64)
+				if err == nil {
+					foundLon = true
+				}
+			}
+		} else {
+			// geohash
+			if len(geoStr) <= geoHashMaxLength {
+				lat, lon = DecodeGeoHash(geoStr)
+				foundLat = true
+				foundLon = true
+			}
+		}
+	}
+
+	// is it a map
+	if l, ok := thing.(map[string]interface{}); ok {
+		if lval, ok := l["lon"]; ok {
+			lon, foundLon = extractNumericVal(lval)
+		} else if lval, ok := l["lng"]; ok {
+			lon, foundLon = extractNumericVal(lval)
+		}
+		if lval, ok := l["lat"]; ok {
+			lat, foundLat = extractNumericVal(lval)
+		}
+	}
+
+	// now try reflection on struct fields
+	if thingVal.Kind() == reflect.Struct {
+		for i := 0; i < thingVal.NumField(); i++ {
+			fieldName := thingTyp.Field(i).Name
+			if strings.HasPrefix(strings.ToLower(fieldName), "lon") {
+				if thingVal.Field(i).CanInterface() {
+					fieldVal := thingVal.Field(i).Interface()
+					lon, foundLon = extractNumericVal(fieldVal)
+				}
+			}
+			if strings.HasPrefix(strings.ToLower(fieldName), "lng") {
+				if thingVal.Field(i).CanInterface() {
+					fieldVal := thingVal.Field(i).Interface()
+					lon, foundLon = extractNumericVal(fieldVal)
+				}
+			}
+			if strings.HasPrefix(strings.ToLower(fieldName), "lat") {
+				if thingVal.Field(i).CanInterface() {
+					fieldVal := thingVal.Field(i).Interface()
+					lat, foundLat = extractNumericVal(fieldVal)
+				}
+			}
+		}
+	}
+
+	// last hope, some interfaces
+	// lon
+	if l, ok := thing.(loner); ok {
+		lon = l.Lon()
+		foundLon = true
+	} else if l, ok := thing.(lnger); ok {
+		lon = l.Lng()
+		foundLon = true
+	}
+	// lat
+	if l, ok := thing.(later); ok {
+		lat = l.Lat()
+		foundLat = true
+	}
+
+	return lon, lat, foundLon && foundLat
+}
+
+// extract numeric value (if possible) and returns a float64
+func extractNumericVal(v interface{}) (float64, bool) {
+	val := reflect.ValueOf(v)
+	if !val.IsValid() {
+		return 0, false
+	}
+	typ := val.Type()
+	switch typ.Kind() {
+	case reflect.Float32, reflect.Float64:
+		return val.Float(), true
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		return float64(val.Int()), true
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+		return float64(val.Uint()), true
+	}
+
+	return 0, false
+}
+
+// various support interfaces which can be used to find lat/lon
+type loner interface {
+	Lon() float64
+}
+
+type later interface {
+	Lat() float64
+}
+
+type lnger interface {
+	Lng() float64
+}

vendor/github.com/blevesearch/bleve/geo/sloppy.go

@@ -0,0 +1,212 @@
+//  Copyright (c) 2017 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package geo
+
+import (
+	"math"
+)
+
+var earthDiameterPerLatitude []float64
+var sinTab []float64
+var cosTab []float64
+var asinTab []float64
+var asinDer1DivF1Tab []float64
+var asinDer2DivF2Tab []float64
+var asinDer3DivF3Tab []float64
+var asinDer4DivF4Tab []float64
+
+const radiusTabsSize = (1 << 10) + 1
+const radiusDelta = (math.Pi / 2) / (radiusTabsSize - 1)
+const radiusIndexer = 1 / radiusDelta
+const sinCosTabsSize = (1 << 11) + 1
+const asinTabsSize = (1 << 13) + 1
+const oneDivF2 = 1 / 2.0
+const oneDivF3 = 1 / 6.0
+const oneDivF4 = 1 / 24.0
+
+// 1.57079632673412561417e+00 first 33 bits of pi/2
+var pio2Hi = math.Float64frombits(0x3FF921FB54400000)
+
+// 6.07710050650619224932e-11 pi/2 - PIO2_HI
+var pio2Lo = math.Float64frombits(0x3DD0B4611A626331)
+
+var asinPio2Hi = math.Float64frombits(0x3FF921FB54442D18) // 1.57079632679489655800e+00
+var asinPio2Lo = math.Float64frombits(0x3C91A62633145C07) // 6.12323399573676603587e-17
+var asinPs0 = math.Float64frombits(0x3fc5555555555555)    //  1.66666666666666657415e-01
+var asinPs1 = math.Float64frombits(0xbfd4d61203eb6f7d)    // -3.25565818622400915405e-01
+var asinPs2 = math.Float64frombits(0x3fc9c1550e884455)    //  2.01212532134862925881e-01
+var asinPs3 = math.Float64frombits(0xbfa48228b5688f3b)    // -4.00555345006794114027e-02
+var asinPs4 = math.Float64frombits(0x3f49efe07501b288)    //  7.91534994289814532176e-04
+var asinPs5 = math.Float64frombits(0x3f023de10dfdf709)    //  3.47933107596021167570e-05
+var asinQs1 = math.Float64frombits(0xc0033a271c8a2d4b)    // -2.40339491173441421878e+00
+var asinQs2 = math.Float64frombits(0x40002ae59c598ac8)    //  2.02094576023350569471e+00
+var asinQs3 = math.Float64frombits(0xbfe6066c1b8d0159)    // -6.88283971605453293030e-01
+var asinQs4 = math.Float64frombits(0x3fb3b8c5b12e9282)    //  7.70381505559019352791e-02
+
+var twoPiHi = 4 * pio2Hi
+var twoPiLo = 4 * pio2Lo
+var sinCosDeltaHi = twoPiHi/sinCosTabsSize - 1
+var sinCosDeltaLo = twoPiLo/sinCosTabsSize - 1
+var sinCosIndexer = 1 / (sinCosDeltaHi + sinCosDeltaLo)
+var sinCosMaxValueForIntModulo = ((math.MaxInt64 >> 9) / sinCosIndexer) * 0.99
+var asinMaxValueForTabs = math.Sin(73.0 * degreesToRadian)
+
+var asinDelta = asinMaxValueForTabs / (asinTabsSize - 1)
+var asinIndexer = 1 / asinDelta
+
+func init() {
+	// initializes the tables used for the sloppy math functions
+
+	// sin and cos
+	sinTab = make([]float64, sinCosTabsSize)
+	cosTab = make([]float64, sinCosTabsSize)
+	sinCosPiIndex := (sinCosTabsSize - 1) / 2
+	sinCosPiMul2Index := 2 * sinCosPiIndex
+	sinCosPiMul05Index := sinCosPiIndex / 2
+	sinCosPiMul15Index := 3 * sinCosPiIndex / 2
+	for i := 0; i < sinCosTabsSize; i++ {
+		// angle: in [0,2*PI].
+		angle := float64(i)*sinCosDeltaHi + float64(i)*sinCosDeltaLo
+		sinAngle := math.Sin(angle)
+		cosAngle := math.Cos(angle)
+		// For indexes corresponding to null cosine or sine, we make sure the value is zero
+		// and not an epsilon. This allows for a much better accuracy for results close to zero.
+		if i == sinCosPiIndex {
+			sinAngle = 0.0
+		} else if i == sinCosPiMul2Index {
+			sinAngle = 0.0
+		} else if i == sinCosPiMul05Index {
+			sinAngle = 0.0
+		} else if i == sinCosPiMul15Index {
+			sinAngle = 0.0
+		}
+		sinTab[i] = sinAngle
+		cosTab[i] = cosAngle
+	}
+
+	// asin
+	asinTab = make([]float64, asinTabsSize)
+	asinDer1DivF1Tab = make([]float64, asinTabsSize)
+	asinDer2DivF2Tab = make([]float64, asinTabsSize)
+	asinDer3DivF3Tab = make([]float64, asinTabsSize)
+	asinDer4DivF4Tab = make([]float64, asinTabsSize)
+	for i := 0; i < asinTabsSize; i++ {
+		// x: in [0,ASIN_MAX_VALUE_FOR_TABS].
+		x := float64(i) * asinDelta
+		asinTab[i] = math.Asin(x)
+		oneMinusXSqInv := 1.0 / (1 - x*x)
+		oneMinusXSqInv05 := math.Sqrt(oneMinusXSqInv)
+		oneMinusXSqInv15 := oneMinusXSqInv05 * oneMinusXSqInv
+		oneMinusXSqInv25 := oneMinusXSqInv15 * oneMinusXSqInv
+		oneMinusXSqInv35 := oneMinusXSqInv25 * oneMinusXSqInv
+		asinDer1DivF1Tab[i] = oneMinusXSqInv05
+		asinDer2DivF2Tab[i] = (x * oneMinusXSqInv15) * oneDivF2
+		asinDer3DivF3Tab[i] = ((1 + 2*x*x) * oneMinusXSqInv25) * oneDivF3
+		asinDer4DivF4Tab[i] = ((5 + 2*x*(2+x*(5-2*x))) * oneMinusXSqInv35) * oneDivF4
+	}
+
+	// earth radius
+	a := 6378137.0
+	b := 6356752.31420
+	a2 := a * a
+	b2 := b * b
+	earthDiameterPerLatitude = make([]float64, radiusTabsSize)
+	earthDiameterPerLatitude[0] = 2.0 * a / 1000
+	earthDiameterPerLatitude[radiusTabsSize-1] = 2.0 * b / 1000
+	for i := 1; i < radiusTabsSize-1; i++ {
+		lat := math.Pi * float64(i) / (2*radiusTabsSize - 1)
+		one := math.Pow(a2*math.Cos(lat), 2)
+		two := math.Pow(b2*math.Sin(lat), 2)
+		three := math.Pow(float64(a)*math.Cos(lat), 2)
+		four := math.Pow(b*math.Sin(lat), 2)
+		radius := math.Sqrt((one + two) / (three + four))
+		earthDiameterPerLatitude[i] = 2 * radius / 1000
+	}
+}
+
+// earthDiameter returns an estimation of the earth's diameter at the specified
+// latitude in kilometers
+func earthDiameter(lat float64) float64 {
+	index := math.Mod(math.Abs(lat)*radiusIndexer+0.5, float64(len(earthDiameterPerLatitude)))
+	if math.IsNaN(index) {
+		return 0
+	}
+	return earthDiameterPerLatitude[int(index)]
+}
+
+var pio2 = math.Pi / 2
+
+func sin(a float64) float64 {
+	return cos(a - pio2)
+}
+
+// cos is a sloppy math (faster) implementation of math.Cos
+func cos(a float64) float64 {
+	if a < 0.0 {
+		a = -a
+	}
+	if a > sinCosMaxValueForIntModulo {
+		return math.Cos(a)
+	}
+	// index: possibly outside tables range.
+	index := int(a*sinCosIndexer + 0.5)
+	delta := (a - float64(index)*sinCosDeltaHi) - float64(index)*sinCosDeltaLo
+	// Making sure index is within tables range.
+	// Last value of each table is the same than first, so we ignore it (tabs size minus one) for modulo.
+	index &= (sinCosTabsSize - 2) // index % (SIN_COS_TABS_SIZE-1)
+	indexCos := cosTab[index]
+	indexSin := sinTab[index]
+	return indexCos + delta*(-indexSin+delta*(-indexCos*oneDivF2+delta*(indexSin*oneDivF3+delta*indexCos*oneDivF4)))
+}
+
+// asin is a sloppy math (faster) implementation of math.Asin
+func asin(a float64) float64 {
+	var negateResult bool
+	if a < 0 {
+		a = -a
+		negateResult = true
+	}
+	if a <= asinMaxValueForTabs {
+		index := int(a*asinIndexer + 0.5)
+		delta := a - float64(index)*asinDelta
+		result := asinTab[index] + delta*(asinDer1DivF1Tab[index]+delta*(asinDer2DivF2Tab[index]+delta*(asinDer3DivF3Tab[index]+delta*asinDer4DivF4Tab[index])))
+		if negateResult {
+			return -result
+		}
+		return result
+	}
+	// value > ASIN_MAX_VALUE_FOR_TABS, or value is NaN
+	// This part is derived from fdlibm.
+	if a < 1 {
+		t := (1.0 - a) * 0.5
+		p := t * (asinPs0 + t*(asinPs1+t*(asinPs2+t*(asinPs3+t*(asinPs4+t+asinPs5)))))
+		q := 1.0 + t*(asinQs1+t*(asinQs2+t*(asinQs3+t*asinQs4)))
+		s := math.Sqrt(t)
+		z := s + s*(p/q)
+		result := asinPio2Hi - ((z + z) - asinPio2Lo)
+		if negateResult {
+			return -result
+		}
+		return result
+	}
+	// value >= 1.0, or value is NaN
+	if a == 1.0 {
+		if negateResult {
+			return -math.Pi / 2
+		}
+		return math.Pi / 2
+	}
+	return math.NaN()
+}

vendor/github.com/blevesearch/bleve/go.mod

@@ -0,0 +1,25 @@
+module github.com/blevesearch/bleve
+
+go 1.13
+
+require (
+	github.com/RoaringBitmap/roaring v0.4.21
+	github.com/blevesearch/blevex v0.0.0-20190916190636-152f0fe5c040
+	github.com/blevesearch/go-porterstemmer v1.0.3
+	github.com/blevesearch/segment v0.9.0
+	github.com/blevesearch/snowballstem v0.9.0
+	github.com/blevesearch/zap/v11 v11.0.7
+	github.com/blevesearch/zap/v12 v12.0.7
+	github.com/couchbase/ghistogram v0.1.0 // indirect
+	github.com/couchbase/moss v0.1.0
+	github.com/couchbase/vellum v1.0.1
+	github.com/golang/protobuf v1.3.2
+	github.com/kljensen/snowball v0.6.0
+	github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563
+	github.com/spf13/cobra v0.0.5
+	github.com/steveyen/gtreap v0.1.0
+	github.com/syndtr/goleveldb v1.0.0
+	github.com/willf/bitset v1.1.10
+	go.etcd.io/bbolt v1.3.4
+	golang.org/x/text v0.3.0
+)

vendor/github.com/blevesearch/bleve/index.go

@@ -0,0 +1,295 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bleve
+
+import (
+	"context"
+
+	"github.com/blevesearch/bleve/document"
+	"github.com/blevesearch/bleve/index"
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/blevesearch/bleve/mapping"
+	"github.com/blevesearch/bleve/size"
+)
+
+// A Batch groups together multiple Index and Delete
+// operations you would like performed at the same
+// time.  The Batch structure is NOT thread-safe.
+// You should only perform operations on a batch
+// from a single thread at a time.  Once batch
+// execution has started, you may not modify it.
+type Batch struct {
+	index    Index
+	internal *index.Batch
+
+	lastDocSize uint64
+	totalSize   uint64
+}
+
+// Index adds the specified index operation to the
+// batch.  NOTE: the bleve Index is not updated
+// until the batch is executed.
+func (b *Batch) Index(id string, data interface{}) error {
+	if id == "" {
+		return ErrorEmptyID
+	}
+	doc := document.NewDocument(id)
+	err := b.index.Mapping().MapDocument(doc, data)
+	if err != nil {
+		return err
+	}
+	b.internal.Update(doc)
+
+	b.lastDocSize = uint64(doc.Size() +
+		len(id) + size.SizeOfString) // overhead from internal
+	b.totalSize += b.lastDocSize
+
+	return nil
+}
+
+func (b *Batch) LastDocSize() uint64 {
+	return b.lastDocSize
+}
+
+func (b *Batch) TotalDocsSize() uint64 {
+	return b.totalSize
+}
+
+// IndexAdvanced adds the specified index operation to the
+// batch which skips the mapping.  NOTE: the bleve Index is not updated
+// until the batch is executed.
+func (b *Batch) IndexAdvanced(doc *document.Document) (err error) {
+	if doc.ID == "" {
+		return ErrorEmptyID
+	}
+	b.internal.Update(doc)
+	return nil
+}
+
+// Delete adds the specified delete operation to the
+// batch.  NOTE: the bleve Index is not updated until
+// the batch is executed.
+func (b *Batch) Delete(id string) {
+	if id != "" {
+		b.internal.Delete(id)
+	}
+}
+
+// SetInternal adds the specified set internal
+// operation to the batch. NOTE: the bleve Index is
+// not updated until the batch is executed.
+func (b *Batch) SetInternal(key, val []byte) {
+	b.internal.SetInternal(key, val)
+}
+
+// DeleteInternal adds the specified delete internal
+// operation to the batch. NOTE: the bleve Index is
+// not updated until the batch is executed.
+func (b *Batch) DeleteInternal(key []byte) {
+	b.internal.DeleteInternal(key)
+}
+
+// Size returns the total number of operations inside the batch
+// including normal index operations and internal operations.
+func (b *Batch) Size() int {
+	return len(b.internal.IndexOps) + len(b.internal.InternalOps)
+}
+
+// String prints a user friendly string representation of what
+// is inside this batch.
+func (b *Batch) String() string {
+	return b.internal.String()
+}
+
+// Reset returns a Batch to the empty state so that it can
+// be re-used in the future.
+func (b *Batch) Reset() {
+	b.internal.Reset()
+	b.lastDocSize = 0
+	b.totalSize = 0
+}
+
+func (b *Batch) Merge(o *Batch) {
+	if o != nil && o.internal != nil {
+		b.internal.Merge(o.internal)
+		if o.LastDocSize() > 0 {
+			b.lastDocSize = o.LastDocSize()
+		}
+		b.totalSize = uint64(b.internal.TotalDocSize())
+	}
+}
+
+func (b *Batch) SetPersistedCallback(f index.BatchCallback) {
+	b.internal.SetPersistedCallback(f)
+}
+
+func (b *Batch) PersistedCallback() index.BatchCallback {
+	return b.internal.PersistedCallback()
+}
+
+// An Index implements all the indexing and searching
+// capabilities of bleve.  An Index can be created
+// using the New() and Open() methods.
+//
+// Index() takes an input value, deduces a DocumentMapping for its type,
+// assigns string paths to its fields or values then applies field mappings on
+// them.
+//
+// The DocumentMapping used to index a value is deduced by the following rules:
+// 1) If value implements mapping.bleveClassifier interface, resolve the mapping
+//    from BleveType().
+// 2) If value implements mapping.Classifier interface, resolve the mapping
+//    from Type().
+// 3) If value has a string field or value at IndexMapping.TypeField.
+// (defaulting to "_type"), use it to resolve the mapping. Fields addressing
+// is described below.
+// 4) If IndexMapping.DefaultType is registered, return it.
+// 5) Return IndexMapping.DefaultMapping.
+//
+// Each field or nested field of the value is identified by a string path, then
+// mapped to one or several FieldMappings which extract the result for analysis.
+//
+// Struct values fields are identified by their "json:" tag, or by their name.
+// Nested fields are identified by prefixing with their parent identifier,
+// separated by a dot.
+//
+// Map values entries are identified by their string key. Entries not indexed
+// by strings are ignored. Entry values are identified recursively like struct
+// fields.
+//
+// Slice and array values are identified by their field name. Their elements
+// are processed sequentially with the same FieldMapping.
+//
+// String, float64 and time.Time values are identified by their field name.
+// Other types are ignored.
+//
+// Each value identifier is decomposed in its parts and recursively address
+// SubDocumentMappings in the tree starting at the root DocumentMapping.  If a
+// mapping is found, all its FieldMappings are applied to the value. If no
+// mapping is found and the root DocumentMapping is dynamic, default mappings
+// are used based on value type and IndexMapping default configurations.
+//
+// Finally, mapped values are analyzed, indexed or stored. See
+// FieldMapping.Analyzer to know how an analyzer is resolved for a given field.
+//
+// Examples:
+//
+//  type Date struct {
+//    Day string `json:"day"`
+//    Month string
+//    Year string
+//  }
+//
+//  type Person struct {
+//    FirstName string `json:"first_name"`
+//    LastName string
+//    BirthDate Date `json:"birth_date"`
+//  }
+//
+// A Person value FirstName is mapped by the SubDocumentMapping at
+// "first_name". Its LastName is mapped by the one at "LastName". The day of
+// BirthDate is mapped to the SubDocumentMapping "day" of the root
+// SubDocumentMapping "birth_date". It will appear as the "birth_date.day"
+// field in the index. The month is mapped to "birth_date.Month".
+type Index interface {
+	// Index analyzes, indexes or stores mapped data fields. Supplied
+	// identifier is bound to analyzed data and will be retrieved by search
+	// requests. See Index interface documentation for details about mapping
+	// rules.
+	Index(id string, data interface{}) error
+	Delete(id string) error
+
+	NewBatch() *Batch
+	Batch(b *Batch) error
+
+	// Document returns specified document or nil if the document is not
+	// indexed or stored.
+	Document(id string) (*document.Document, error)
+	// DocCount returns the number of documents in the index.
+	DocCount() (uint64, error)
+
+	Search(req *SearchRequest) (*SearchResult, error)
+	SearchInContext(ctx context.Context, req *SearchRequest) (*SearchResult, error)
+
+	Fields() ([]string, error)
+
+	FieldDict(field string) (index.FieldDict, error)
+	FieldDictRange(field string, startTerm []byte, endTerm []byte) (index.FieldDict, error)
+	FieldDictPrefix(field string, termPrefix []byte) (index.FieldDict, error)
+
+	Close() error
+
+	Mapping() mapping.IndexMapping
+
+	Stats() *IndexStat
+	StatsMap() map[string]interface{}
+
+	GetInternal(key []byte) ([]byte, error)
+	SetInternal(key, val []byte) error
+	DeleteInternal(key []byte) error
+
+	// Name returns the name of the index (by default this is the path)
+	Name() string
+	// SetName lets you assign your own logical name to this index
+	SetName(string)
+
+	// Advanced returns the indexer and data store, exposing lower level
+	// methods to enumerate records and access data.
+	Advanced() (index.Index, store.KVStore, error)
+}
+
+// New index at the specified path, must not exist.
+// The provided mapping will be used for all
+// Index/Search operations.
+func New(path string, mapping mapping.IndexMapping) (Index, error) {
+	return newIndexUsing(path, mapping, Config.DefaultIndexType, Config.DefaultKVStore, nil)
+}
+
+// NewMemOnly creates a memory-only index.
+// The contents of the index is NOT persisted,
+// and will be lost once closed.
+// The provided mapping will be used for all
+// Index/Search operations.
+func NewMemOnly(mapping mapping.IndexMapping) (Index, error) {
+	return newIndexUsing("", mapping, Config.DefaultIndexType, Config.DefaultMemKVStore, nil)
+}
+
+// NewUsing creates index at the specified path,
+// which must not already exist.
+// The provided mapping will be used for all
+// Index/Search operations.
+// The specified index type will be used.
+// The specified kvstore implementation will be used
+// and the provided kvconfig will be passed to its
+// constructor. Note that currently the values of kvconfig must
+// be able to be marshaled and unmarshaled using the encoding/json library (used
+// when reading/writing the index metadata file).
+func NewUsing(path string, mapping mapping.IndexMapping, indexType string, kvstore string, kvconfig map[string]interface{}) (Index, error) {
+	return newIndexUsing(path, mapping, indexType, kvstore, kvconfig)
+}
+
+// Open index at the specified path, must exist.
+// The mapping used when it was created will be used for all Index/Search operations.
+func Open(path string) (Index, error) {
+	return openIndexUsing(path, nil)
+}
+
+// OpenUsing opens index at the specified path, must exist.
+// The mapping used when it was created will be used for all Index/Search operations.
+// The provided runtimeConfig can override settings
+// persisted when the kvstore was created.
+func OpenUsing(path string, runtimeConfig map[string]interface{}) (Index, error) {
+	return openIndexUsing(path, runtimeConfig)
+}

vendor/github.com/blevesearch/bleve/index/analysis.go

@@ -0,0 +1,110 @@
+//  Copyright (c) 2015 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package index
+
+import (
+	"reflect"
+
+	"github.com/blevesearch/bleve/analysis"
+	"github.com/blevesearch/bleve/document"
+	"github.com/blevesearch/bleve/size"
+)
+
+var reflectStaticSizeAnalysisResult int
+
+func init() {
+	var ar AnalysisResult
+	reflectStaticSizeAnalysisResult = int(reflect.TypeOf(ar).Size())
+}
+
+type IndexRow interface {
+	KeySize() int
+	KeyTo([]byte) (int, error)
+	Key() []byte
+
+	ValueSize() int
+	ValueTo([]byte) (int, error)
+	Value() []byte
+}
+
+type AnalysisResult struct {
+	DocID string
+	Rows  []IndexRow
+
+	// scorch
+	Document *document.Document
+	Analyzed []analysis.TokenFrequencies
+	Length   []int
+}
+
+func (a *AnalysisResult) Size() int {
+	rv := reflectStaticSizeAnalysisResult
+	for _, analyzedI := range a.Analyzed {
+		rv += analyzedI.Size()
+	}
+	rv += len(a.Length) * size.SizeOfInt
+	return rv
+}
+
+type AnalysisWork struct {
+	i  Index
+	d  *document.Document
+	rc chan *AnalysisResult
+}
+
+func NewAnalysisWork(i Index, d *document.Document, rc chan *AnalysisResult) *AnalysisWork {
+	return &AnalysisWork{
+		i:  i,
+		d:  d,
+		rc: rc,
+	}
+}
+
+type AnalysisQueue struct {
+	queue chan *AnalysisWork
+	done  chan struct{}
+}
+
+func (q *AnalysisQueue) Queue(work *AnalysisWork) {
+	q.queue <- work
+}
+
+func (q *AnalysisQueue) Close() {
+	close(q.done)
+}
+
+func NewAnalysisQueue(numWorkers int) *AnalysisQueue {
+	rv := AnalysisQueue{
+		queue: make(chan *AnalysisWork),
+		done:  make(chan struct{}),
+	}
+	for i := 0; i < numWorkers; i++ {
+		go AnalysisWorker(rv)
+	}
+	return &rv
+}
+
+func AnalysisWorker(q AnalysisQueue) {
+	// read work off the queue
+	for {
+		select {
+		case <-q.done:
+			return
+		case w := <-q.queue:
+			r := w.i.Analyze(w.d)
+			w.rc <- r
+		}
+	}
+}

vendor/github.com/blevesearch/bleve/index/field_cache.go

@@ -0,0 +1,88 @@
+//  Copyright (c) 2015 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package index
+
+import (
+	"sync"
+)
+
+type FieldCache struct {
+	fieldIndexes   map[string]uint16
+	indexFields    []string
+	lastFieldIndex int
+	mutex          sync.RWMutex
+}
+
+func NewFieldCache() *FieldCache {
+	return &FieldCache{
+		fieldIndexes:   make(map[string]uint16),
+		lastFieldIndex: -1,
+	}
+}
+
+func (f *FieldCache) AddExisting(field string, index uint16) {
+	f.mutex.Lock()
+	f.addLOCKED(field, index)
+	f.mutex.Unlock()
+}
+
+func (f *FieldCache) addLOCKED(field string, index uint16) uint16 {
+	f.fieldIndexes[field] = index
+	if len(f.indexFields) < int(index)+1 {
+		prevIndexFields := f.indexFields
+		f.indexFields = make([]string, int(index)+16)
+		copy(f.indexFields, prevIndexFields)
+	}
+	f.indexFields[int(index)] = field
+	if int(index) > f.lastFieldIndex {
+		f.lastFieldIndex = int(index)
+	}
+	return index
+}
+
+// FieldNamed returns the index of the field, and whether or not it existed
+// before this call.  if createIfMissing is true, and new field index is assigned
+// but the second return value will still be false
+func (f *FieldCache) FieldNamed(field string, createIfMissing bool) (uint16, bool) {
+	f.mutex.RLock()
+	if index, ok := f.fieldIndexes[field]; ok {
+		f.mutex.RUnlock()
+		return index, true
+	} else if !createIfMissing {
+		f.mutex.RUnlock()
+		return 0, false
+	}
+	// trade read lock for write lock
+	f.mutex.RUnlock()
+	f.mutex.Lock()
+	// need to check again with write lock
+	if index, ok := f.fieldIndexes[field]; ok {
+		f.mutex.Unlock()
+		return index, true
+	}
+	// assign next field id
+	index := f.addLOCKED(field, uint16(f.lastFieldIndex+1))
+	f.mutex.Unlock()
+	return index, false
+}
+
+func (f *FieldCache) FieldIndexed(index uint16) (field string) {
+	f.mutex.RLock()
+	if int(index) < len(f.indexFields) {
+		field = f.indexFields[int(index)]
+	}
+	f.mutex.RUnlock()
+	return field
+}

vendor/github.com/blevesearch/bleve/index/index.go

@@ -0,0 +1,369 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package index
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"reflect"
+
+	"github.com/blevesearch/bleve/document"
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/blevesearch/bleve/size"
+)
+
+var reflectStaticSizeTermFieldDoc int
+var reflectStaticSizeTermFieldVector int
+
+func init() {
+	var tfd TermFieldDoc
+	reflectStaticSizeTermFieldDoc = int(reflect.TypeOf(tfd).Size())
+	var tfv TermFieldVector
+	reflectStaticSizeTermFieldVector = int(reflect.TypeOf(tfv).Size())
+}
+
+var ErrorUnknownStorageType = fmt.Errorf("unknown storage type")
+
+type Index interface {
+	Open() error
+	Close() error
+
+	Update(doc *document.Document) error
+	Delete(id string) error
+	Batch(batch *Batch) error
+
+	SetInternal(key, val []byte) error
+	DeleteInternal(key []byte) error
+
+	// Reader returns a low-level accessor on the index data. Close it to
+	// release associated resources.
+	Reader() (IndexReader, error)
+
+	Stats() json.Marshaler
+	StatsMap() map[string]interface{}
+
+	Analyze(d *document.Document) *AnalysisResult
+
+	Advanced() (store.KVStore, error)
+}
+
+type DocumentFieldTermVisitor func(field string, term []byte)
+
+type IndexReader interface {
+	TermFieldReader(term []byte, field string, includeFreq, includeNorm, includeTermVectors bool) (TermFieldReader, error)
+
+	// DocIDReader returns an iterator over all doc ids
+	// The caller must close returned instance to release associated resources.
+	DocIDReaderAll() (DocIDReader, error)
+
+	DocIDReaderOnly(ids []string) (DocIDReader, error)
+
+	FieldDict(field string) (FieldDict, error)
+
+	// FieldDictRange is currently defined to include the start and end terms
+	FieldDictRange(field string, startTerm []byte, endTerm []byte) (FieldDict, error)
+	FieldDictPrefix(field string, termPrefix []byte) (FieldDict, error)
+
+	Document(id string) (*document.Document, error)
+	DocumentVisitFieldTerms(id IndexInternalID, fields []string, visitor DocumentFieldTermVisitor) error
+
+	DocValueReader(fields []string) (DocValueReader, error)
+
+	Fields() ([]string, error)
+
+	GetInternal(key []byte) ([]byte, error)
+
+	DocCount() (uint64, error)
+
+	ExternalID(id IndexInternalID) (string, error)
+	InternalID(id string) (IndexInternalID, error)
+
+	DumpAll() chan interface{}
+	DumpDoc(id string) chan interface{}
+	DumpFields() chan interface{}
+
+	Close() error
+}
+
+// The Regexp interface defines the subset of the regexp.Regexp API
+// methods that are used by bleve indexes, allowing callers to pass in
+// alternate implementations.
+type Regexp interface {
+	FindStringIndex(s string) (loc []int)
+
+	LiteralPrefix() (prefix string, complete bool)
+
+	String() string
+}
+
+type IndexReaderRegexp interface {
+	FieldDictRegexp(field string, regex string) (FieldDict, error)
+}
+
+type IndexReaderFuzzy interface {
+	FieldDictFuzzy(field string, term string, fuzziness int, prefix string) (FieldDict, error)
+}
+
+type IndexReaderOnly interface {
+	FieldDictOnly(field string, onlyTerms [][]byte, includeCount bool) (FieldDict, error)
+}
+
+type IndexReaderContains interface {
+	FieldDictContains(field string) (FieldDictContains, error)
+}
+
+// FieldTerms contains the terms used by a document, keyed by field
+type FieldTerms map[string][]string
+
+// FieldsNotYetCached returns a list of fields not yet cached out of a larger list of fields
+func (f FieldTerms) FieldsNotYetCached(fields []string) []string {
+	rv := make([]string, 0, len(fields))
+	for _, field := range fields {
+		if _, ok := f[field]; !ok {
+			rv = append(rv, field)
+		}
+	}
+	return rv
+}
+
+// Merge will combine two FieldTerms
+// it assumes that the terms lists are complete (thus do not need to be merged)
+// field terms from the other list always replace the ones in the receiver
+func (f FieldTerms) Merge(other FieldTerms) {
+	for field, terms := range other {
+		f[field] = terms
+	}
+}
+
+type TermFieldVector struct {
+	Field          string
+	ArrayPositions []uint64
+	Pos            uint64
+	Start          uint64
+	End            uint64
+}
+
+func (tfv *TermFieldVector) Size() int {
+	return reflectStaticSizeTermFieldVector + size.SizeOfPtr +
+		len(tfv.Field) + len(tfv.ArrayPositions)*size.SizeOfUint64
+}
+
+// IndexInternalID is an opaque document identifier interal to the index impl
+type IndexInternalID []byte
+
+func (id IndexInternalID) Equals(other IndexInternalID) bool {
+	return id.Compare(other) == 0
+}
+
+func (id IndexInternalID) Compare(other IndexInternalID) int {
+	return bytes.Compare(id, other)
+}
+
+type TermFieldDoc struct {
+	Term    string
+	ID      IndexInternalID
+	Freq    uint64
+	Norm    float64
+	Vectors []*TermFieldVector
+}
+
+func (tfd *TermFieldDoc) Size() int {
+	sizeInBytes := reflectStaticSizeTermFieldDoc + size.SizeOfPtr +
+		len(tfd.Term) + len(tfd.ID)
+
+	for _, entry := range tfd.Vectors {
+		sizeInBytes += entry.Size()
+	}
+
+	return sizeInBytes
+}
+
+// Reset allows an already allocated TermFieldDoc to be reused
+func (tfd *TermFieldDoc) Reset() *TermFieldDoc {
+	// remember the []byte used for the ID
+	id := tfd.ID
+	vectors := tfd.Vectors
+	// idiom to copy over from empty TermFieldDoc (0 allocations)
+	*tfd = TermFieldDoc{}
+	// reuse the []byte already allocated (and reset len to 0)
+	tfd.ID = id[:0]
+	tfd.Vectors = vectors[:0]
+	return tfd
+}
+
+// TermFieldReader is the interface exposing the enumeration of documents
+// containing a given term in a given field. Documents are returned in byte
+// lexicographic order over their identifiers.
+type TermFieldReader interface {
+	// Next returns the next document containing the term in this field, or nil
+	// when it reaches the end of the enumeration.  The preAlloced TermFieldDoc
+	// is optional, and when non-nil, will be used instead of allocating memory.
+	Next(preAlloced *TermFieldDoc) (*TermFieldDoc, error)
+
+	// Advance resets the enumeration at specified document or its immediate
+	// follower.
+	Advance(ID IndexInternalID, preAlloced *TermFieldDoc) (*TermFieldDoc, error)
+
+	// Count returns the number of documents contains the term in this field.
+	Count() uint64
+	Close() error
+
+	Size() int
+}
+
+type DictEntry struct {
+	Term  string
+	Count uint64
+}
+
+type FieldDict interface {
+	Next() (*DictEntry, error)
+	Close() error
+}
+
+type FieldDictContains interface {
+	Contains(key []byte) (bool, error)
+}
+
+// DocIDReader is the interface exposing enumeration of documents identifiers.
+// Close the reader to release associated resources.
+type DocIDReader interface {
+	// Next returns the next document internal identifier in the natural
+	// index order, nil when the end of the sequence is reached.
+	Next() (IndexInternalID, error)
+
+	// Advance resets the iteration to the first internal identifier greater than
+	// or equal to ID. If ID is smaller than the start of the range, the iteration
+	// will start there instead. If ID is greater than or equal to the end of
+	// the range, Next() call will return io.EOF.
+	Advance(ID IndexInternalID) (IndexInternalID, error)
+
+	Size() int
+
+	Close() error
+}
+
+type BatchCallback func(error)
+
+type Batch struct {
+	IndexOps          map[string]*document.Document
+	InternalOps       map[string][]byte
+	persistedCallback BatchCallback
+}
+
+func NewBatch() *Batch {
+	return &Batch{
+		IndexOps:    make(map[string]*document.Document),
+		InternalOps: make(map[string][]byte),
+	}
+}
+
+func (b *Batch) Update(doc *document.Document) {
+	b.IndexOps[doc.ID] = doc
+}
+
+func (b *Batch) Delete(id string) {
+	b.IndexOps[id] = nil
+}
+
+func (b *Batch) SetInternal(key, val []byte) {
+	b.InternalOps[string(key)] = val
+}
+
+func (b *Batch) DeleteInternal(key []byte) {
+	b.InternalOps[string(key)] = nil
+}
+
+func (b *Batch) SetPersistedCallback(f BatchCallback) {
+	b.persistedCallback = f
+}
+
+func (b *Batch) PersistedCallback() BatchCallback {
+	return b.persistedCallback
+}
+
+func (b *Batch) String() string {
+	rv := fmt.Sprintf("Batch (%d ops, %d internal ops)\n", len(b.IndexOps), len(b.InternalOps))
+	for k, v := range b.IndexOps {
+		if v != nil {
+			rv += fmt.Sprintf("\tINDEX - '%s'\n", k)
+		} else {
+			rv += fmt.Sprintf("\tDELETE - '%s'\n", k)
+		}
+	}
+	for k, v := range b.InternalOps {
+		if v != nil {
+			rv += fmt.Sprintf("\tSET INTERNAL - '%s'\n", k)
+		} else {
+			rv += fmt.Sprintf("\tDELETE INTERNAL - '%s'\n", k)
+		}
+	}
+	return rv
+}
+
+func (b *Batch) Reset() {
+	b.IndexOps = make(map[string]*document.Document)
+	b.InternalOps = make(map[string][]byte)
+	b.persistedCallback = nil
+}
+
+func (b *Batch) Merge(o *Batch) {
+	for k, v := range o.IndexOps {
+		b.IndexOps[k] = v
+	}
+	for k, v := range o.InternalOps {
+		b.InternalOps[k] = v
+	}
+}
+
+func (b *Batch) TotalDocSize() int {
+	var s int
+	for k, v := range b.IndexOps {
+		if v != nil {
+			s += v.Size() + size.SizeOfString
+		}
+		s += len(k)
+	}
+	return s
+}
+
+// Optimizable represents an optional interface that implementable by
+// optimizable resources (e.g., TermFieldReaders, Searchers).  These
+// optimizable resources are provided the same OptimizableContext
+// instance, so that they can coordinate via dynamic interface
+// casting.
+type Optimizable interface {
+	Optimize(kind string, octx OptimizableContext) (OptimizableContext, error)
+}
+
+// Represents a result of optimization -- see the Finish() method.
+type Optimized interface{}
+
+type OptimizableContext interface {
+	// Once all the optimzable resources have been provided the same
+	// OptimizableContext instance, the optimization preparations are
+	// finished or completed via the Finish() method.
+	//
+	// Depending on the optimization being performed, the Finish()
+	// method might return a non-nil Optimized instance.  For example,
+	// the Optimized instance might represent an optimized
+	// TermFieldReader instance.
+	Finish() (Optimized, error)
+}
+
+type DocValueReader interface {
+	VisitDocValues(id IndexInternalID, visitor DocumentFieldTermVisitor) error
+}

vendor/github.com/blevesearch/bleve/index/scorch/README.md

@@ -0,0 +1,367 @@
+# scorch
+
+## Definitions
+
+Batch
+- A collection of Documents to mutate in the index.
+
+Document
+- Has a unique identifier (arbitrary bytes).
+- Is comprised of a list of fields.
+
+Field
+- Has a name (string).
+- Has a type (text, number, date, geopoint).
+- Has a value (depending on type).
+- Can be indexed, stored, or both.
+- If indexed, can be analyzed.
+-m If indexed, can optionally store term vectors.
+
+## Scope
+
+Scorch *MUST* implement the bleve.index API without requiring any changes to this API.
+
+Scorch *MAY* introduce new interfaces, which can be discovered to allow use of new capabilities not in the current API.
+
+## Implementation
+
+The scorch implementation starts with the concept of a segmented index.
+
+A segment is simply a slice, subset, or portion of the entire index.  A segmented index is one which is composed of one or more segments.  Although segments are created in a particular order, knowing this ordering is not required to achieve correct semantics when querying.  Because there is no ordering, this means that when searching an index, you can (and should) search all the segments concurrently.
+
+### Internal Wrapper
+
+In order to accommodate the existing APIs while also improving the implementation, the scorch implementation includes some wrapper functionality that must be described.
+
+#### \_id field
+
+In scorch, field 0 is prearranged to be named \_id.  All documents have a value for this field, which is the documents external identifier.  In this version the field *MUST* be both indexed AND stored.  The scorch wrapper adds this field, as it will not be present in the Document from the calling bleve code.
+
+NOTE: If a document already contains a field \_id, it will be replaced.  If this is problematic, the caller must ensure such a scenario does not happen.
+
+### Proposed Structures
+
+```
+type Segment interface {
+
+  Dictionary(field string) TermDictionary
+
+}
+
+type TermDictionary interface {
+
+  PostingsList(term string, excluding PostingsList) PostingsList
+
+}
+
+type PostingsList interface {
+
+  Next() Posting
+
+  And(other PostingsList) PostingsList
+  Or(other PostingsList) PostingsList
+
+}
+
+type Posting interface {
+  Number() uint64
+
+  Frequency() uint64
+  Norm() float64
+
+  Locations() Locations
+}
+
+type Locations interface {
+  Start() uint64
+  End() uint64
+  Pos() uint64
+  ArrayPositions() ...
+}
+
+type DeletedDocs {
+
+}
+
+type SegmentSnapshot struct {
+  segment Segment
+  deleted PostingsList
+}
+
+type IndexSnapshot struct {
+  segment []SegmentSnapshot
+}
+```
+**What about errors?**
+**What about memory mgmnt or context?**
+**Postings List separate iterator to separate stateful from stateless**
+### Mutating the Index
+
+The bleve.index API has methods for directly making individual mutations (Update/Delete/SetInternal/DeleteInternal), however for this first implementation, we assume that all of these calls can simply be turned into a Batch of size 1.  This may be highly inefficient, but it will be correct.  This decision is made based on the fact that Couchbase FTS always uses Batches.
+
+NOTE: As a side-effect of this decision, it should be clear that performance tuning may depend on the batch size, which may in-turn require changes in FTS.
+
+From this point forward, only Batch mutations will be discussed.
+
+Sequence of Operations:
+
+1.  For each document in the batch, search through all existing segments.  The goal is to build up a per-segment bitset which tells us which documents in that segment are obsoleted by the addition of the new segment we're currently building.  NOTE: we're not ready for this change to take effect yet, so rather than this operation mutating anything, they simply return bitsets, which we can apply later.  Logically, this is something like:
+
+    ```
+      foreach segment {
+        dict := segment.Dictionary("\_id")
+        postings := empty postings list
+        foreach docID {
+          postings = postings.Or(dict.PostingsList(docID, nil))
+        }
+      }
+    ```
+
+    NOTE: it is illustrated above as nested for loops, but some or all of these could be concurrently.  The end result is that for each segment, we have (possibly empty) bitset.
+
+2.  Also concurrent with 1, the documents in the batch are analyzed.  This analysis proceeds using the existing analyzer pool.
+
+3. (after 2 completes) Analyzed documents are fed into a function which builds a new Segment representing this information.
+
+4.  We now have everything we need to update the state of the system to include this new snapshot.
+
+  -  Acquire a lock
+  -  Create a new IndexSnapshot
+  -  For each SegmentSnapshot in the IndexSnapshot, take the deleted PostingsList and OR it with the new postings list for this Segment.  Construct a new SegmentSnapshot for the segment using this new deleted PostingsList.  Append this SegmentSnapshot to the IndexSnapshot.
+  -  Create a new SegmentSnapshot wrapping our new segment with nil deleted docs.
+  -  Append the new SegmentSnapshot to the IndexSnapshot
+  -  Release the lock
+
+An ASCII art example:
+  ```
+  0 - Empty Index
+
+  No segments
+
+  IndexSnapshot
+    segments []
+    deleted []
+
+
+  1 - Index Batch [ A B C ]
+
+  segment       0
+  numbers   [ 1 2 3 ]
+  \_id      [ A B C ]
+
+  IndexSnapshot
+    segments [ 0 ]
+    deleted [ nil ]
+
+
+  2 - Index Batch [ B' ]
+
+  segment       0           1
+  numbers   [ 1 2 3 ]     [ 1 ]
+  \_id      [ A B C ]     [ B ]
+
+  Compute bitset segment-0-deleted-by-1:
+            [ 0 1 0 ]
+
+  OR it with previous (nil) (call it 0-1)
+            [ 0 1 0 ]
+
+  IndexSnapshot
+    segments [  0    1 ]
+    deleted  [ 0-1 nil ]
+
+  3 - Index Batch [ C' ]
+
+    segment       0           1      2
+    numbers   [ 1 2 3 ]     [ 1 ]  [ 1 ]
+    \_id      [ A B C ]     [ B ]  [ C ]
+
+    Compute bitset segment-0-deleted-by-2:
+              [ 0 0 1 ]
+
+    OR it with previous ([ 0 1 0 ]) (call it 0-12)
+              [ 0 1 1 ]
+
+  Compute bitset segment-1-deleted-by-2:
+              [ 0 ]
+
+  OR it with previous (nil)
+              still just nil
+
+
+    IndexSnapshot
+      segments [  0    1    2 ]
+      deleted  [ 0-12 nil  nil ]
+  ```
+
+**is there opportunity to stop early when doc is found in one segment**
+**also, more efficient way to find bits for long lists of ids?**
+
+### Searching
+
+In the bleve.index API all searching starts by getting an IndexReader, which represents a snapshot of the index at a point in time.
+
+As described in the section above, our index implementation maintains a pointer to the current IndexSnapshot.  When a caller gets an IndexReader, they get a copy of this pointer, and can use it as long as they like.  The IndexSnapshot contains SegmentSnapshots, which only contain pointers to immutable segments.  The deleted posting lists associated with a segment change over time, but the particular deleted posting list in YOUR snapshot is immutable.  This gives a stable view of the data.
+
+#### Term Search
+
+Term search is the only searching primitive exposed in today's bleve.index API.  This ultimately could limit our ability to take advantage of the indexing improvements, but it also means it will be easier to get a first version of this working.
+
+A term search for term T in field F will look something like this:
+
+```
+  searchResultPostings = empty
+  foreach segment {
+    dict := segment.Dictionary(F)
+    segmentResultPostings = dict.PostingsList(T, segmentSnapshotDeleted)
+    // make segmentLocal numbers into global numbers, and flip bits in searchResultPostings
+  }
+```
+
+The searchResultPostings will be a new implementation of the TermFieldReader inteface.
+
+As a reminder this interface is:
+
+```
+// TermFieldReader is the interface exposing the enumeration of documents
+// containing a given term in a given field. Documents are returned in byte
+// lexicographic order over their identifiers.
+type TermFieldReader interface {
+	// Next returns the next document containing the term in this field, or nil
+	// when it reaches the end of the enumeration.  The preAlloced TermFieldDoc
+	// is optional, and when non-nil, will be used instead of allocating memory.
+	Next(preAlloced *TermFieldDoc) (*TermFieldDoc, error)
+
+	// Advance resets the enumeration at specified document or its immediate
+	// follower.
+	Advance(ID IndexInternalID, preAlloced *TermFieldDoc) (*TermFieldDoc, error)
+
+	// Count returns the number of documents contains the term in this field.
+	Count() uint64
+	Close() error
+}
+```
+
+At first glance this appears problematic, we have no way to return documents in order of their identifiers.  But it turns out the wording of this perhaps too strong, or a bit ambiguous.  Originally, this referred to the external identifiers, but with the introduction of a distinction between internal/external identifiers, returning them in order of their internal identifiers is also acceptable.  **ASIDE**: the reason for this is that most callers just use Next() and literally don't care what the order is, they could be in any order and it would be fine.  There is only one search that cares and that is the ConjunctionSearcher, which relies on Next/Advance having very specific semantics.  Later in this document we will have a proposal to split into multiple interfaces:
+
+-  The weakest interface, only supports Next() no ordering at all.
+-  Ordered, supporting Advance()
+-  And/Or'able capable of internally efficiently doing these ops with like interfaces (if not capable then can always fall back to external walking)
+
+But, the good news is that we don't even have to do that for our first implementation.  As long as the global numbers we use for internal identifiers are consistent within this IndexSnapshot, then Next() will be ordered by ascending document number, and Advance() will still work correctly.
+
+NOTE: there is another place where we rely on the ordering of these hits, and that is in the "\_id" sort order.  Previously this was the natural order, and a NOOP for the collector, now it must be implemented by actually sorting on the "\_id" field.  We probably should introduce at least a marker interface to detect this.
+
+An ASCII art example:
+
+```
+Let's start with the IndexSnapshot we ended with earlier:
+
+3 - Index Batch [ C' ]
+
+  segment       0           1      2
+  numbers   [ 1 2 3 ]     [ 1 ]  [ 1 ]
+  \_id      [ A B C ]     [ B ]  [ C ]
+
+  Compute bitset segment-0-deleted-by-2:
+            [ 0 0 1 ]
+
+  OR it with previous ([ 0 1 0 ]) (call it 0-12)
+            [ 0 1 1 ]
+
+Compute bitset segment-1-deleted-by-2:
+            [ 0 0 0 ]
+
+OR it with previous (nil)
+            still just nil
+
+
+  IndexSnapshot
+    segments [  0    1    2 ]
+    deleted  [ 0-12 nil  nil ]
+
+Now let's search for the term 'cat' in the field 'desc' and let's assume that Document C (both versions) would match it.
+
+Concurrently:
+
+  - Segment 0
+   - Get Term Dictionary For Field 'desc'
+   - From it get Postings List for term 'cat' EXCLUDING 0-12
+   - raw segment matches [ 0 0 1 ] but excluding [ 0 1 1 ] gives [ 0 0 0 ]
+  - Segment 1
+   - Get Term Dictionary For Field 'desc'
+   - From it get Postings List for term 'cat' excluding nil
+   - [ 0 ]
+  - Segment 2
+   - Get Term Dictionary For Field 'desc'
+   - From it get Postings List for term 'cat' excluding nil
+   - [ 1 ]
+
+Map local bitsets into global number space (global meaning cross-segment but still unique to this snapshot)
+
+IndexSnapshot already should have mapping something like:
+0 - Offset 0
+1 - Offset 3 (because segment 0 had 3 docs)
+2 - Offset 4 (because segment 1 had 1 doc)
+
+This maps to search result bitset:
+
+[ 0 0 0 0 1]
+
+Caller would call Next() and get doc number 5 (assuming 1 based indexing for now)
+
+Caller could then ask to get term locations, stored fields, external doc ID for document number 5.  Internally in the IndexSnapshot, we can now convert that back, and realize doc number 5 comes from segment 2, 5-4=1 so we're looking for doc number 1 in segment 2.  That happens to be C...
+
+```
+
+#### Future improvements
+
+In the future, interfaces to detect these non-serially operating TermFieldReaders could expose their own And() and Or() up to the higher level Conjunction/Disjunction searchers.  Doing this alone offers some win, but also means there would be greater burden on the Searcher code rewriting logical expressions for maximum performance.
+
+Another related topic is that of peak memory usage.  With serially operating TermFieldReaders it was necessary to start them all at the same time and operate in unison.  However, with these non-serially operating TermFieldReaders we have the option of doing a few at a time, consolidating them, dispoting the intermediaries, and then doing a few more.  For very complex queries with many clauses this could reduce peak memory usage.
+
+
+### Memory Tracking
+
+All segments must be able to produce two statistics, an estimate of their explicit memory usage, and their actual size on disk (if any).  For in-memory segments, disk usage could be zero, and the memory usage represents the entire information content.  For mmap-based disk segments, the memory could be as low as the size of tracking structure itself (say just a few pointers).
+
+This would allow the implementation to throttle or block incoming mutations when a threshold memory usage has (or would be) exceeded.
+
+### Persistence
+
+Obviously, we want to support (but maybe not require) asynchronous persistence of segments.  My expectation is that segments are initially built in memory.  At some point they are persisted to disk.  This poses some interesting challenges.
+
+At runtime, the state of an index (it's IndexSnapshot) is not only the contents of the segments, but also the bitmasks of deleted documents.  These bitmasks indirectly encode an ordering in which the segments were added.  The reason is that the bitmasks encode which items have been obsoleted by other (subsequent or more future) segments.  In the runtime implementation we compute bitmask deltas and then merge them at the same time we bring the new segment in.  One idea is that we could take a similar approach on disk.  When we persist a segment, we persist the bitmask deltas of segments known to exist at that time, and eventually these can get merged up into a base segment deleted bitmask.
+
+This also relates to the topic rollback, addressed next...
+
+
+### Rollback
+
+One desirable property in the Couchbase ecosystem is the ability to rollback to some previous (though typically not long ago) state.  One idea for keeping this property in this design is to protect some of the most recent segments from merging.  Then, if necessary, they could be "undone" to reveal previous states of the system.  In these scenarios "undone" has to properly undo the deleted bitmasks on the other segments.  Again, the current thinking is that rather than "undo" anything, it could be work that was deferred in the first place, thus making it easier to logically undo.
+
+Another possibly related approach would be to tie this into our existing snapshot mechanism.  Perhaps simulating a slow reader (holding onto index snapshots) for some period of time, can be the mechanism to achieve the desired end goal.
+
+
+### Internal Storage
+
+The bleve.index API has support for "internal storage".  The ability to store information under a separate name space.
+
+This is not used for high volume storage, so it is tempting to think we could just put a small k/v store alongside the rest of the index.  But, the reality is that this storage is used to maintain key information related to the rollback scenario.  Because of this, its crucial that ordering and overwriting of key/value pairs correspond with actual segment persistence in the index.  Based on this, I believe its important to put the internal key/value pairs inside the segments themselves.  But, this also means that they must follow a similar "deleted" bitmask approach to obsolete values in older segments.  But, this also seems to substantially increase the complexity of the solution because of the separate name space, it would appear to require its own bitmask.  Further keys aren't numeric, which then implies yet another mapping from internal key to number, etc.
+
+More thought is required here.
+
+### Merging
+
+The segmented index approach requires merging to prevent the number of segments from growing too large.
+
+Recent experience with LSMs has taught us that having the correct merge strategy can make a huge difference in the overall performance of the system.  In particular, a simple merge strategy which merges segments too aggressively can lead to high write amplification and unnecessarily rendering cached data useless.
+
+A few simple principles have been identified.
+
+- Roughly we merge multiple smaller segments into a single larger one.
+- The larger a segment gets the less likely we should be to ever merge it.
+- Segments with large numbers of deleted/obsoleted items are good candidates as the merge will result in a space savings.
+- Segments with all items deleted/obsoleted can be dropped.
+
+Merging of a segment should be able to proceed even if that segment is held by an ongoing snapshot, it should only delay the removal of it.

vendor/github.com/blevesearch/bleve/index/scorch/event.go

@@ -0,0 +1,56 @@
+//  Copyright (c) 2018 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package scorch
+
+import "time"
+
+// RegistryAsyncErrorCallbacks should be treated as read-only after
+// process init()'ialization.
+var RegistryAsyncErrorCallbacks = map[string]func(error){}
+
+// RegistryEventCallbacks should be treated as read-only after
+// process init()'ialization.
+var RegistryEventCallbacks = map[string]func(Event){}
+
+// Event represents the information provided in an OnEvent() callback.
+type Event struct {
+	Kind     EventKind
+	Scorch   *Scorch
+	Duration time.Duration
+}
+
+// EventKind represents an event code for OnEvent() callbacks.
+type EventKind int
+
+// EventKindCloseStart is fired when a Scorch.Close() has begun.
+var EventKindCloseStart = EventKind(1)
+
+// EventKindClose is fired when a scorch index has been fully closed.
+var EventKindClose = EventKind(2)
+
+// EventKindMergerProgress is fired when the merger has completed a
+// round of merge processing.
+var EventKindMergerProgress = EventKind(3)
+
+// EventKindPersisterProgress is fired when the persister has completed
+// a round of persistence processing.
+var EventKindPersisterProgress = EventKind(4)
+
+// EventKindBatchIntroductionStart is fired when Batch() is invoked which
+// introduces a new segment.
+var EventKindBatchIntroductionStart = EventKind(5)
+
+// EventKindBatchIntroduction is fired when Batch() completes.
+var EventKindBatchIntroduction = EventKind(6)

vendor/github.com/blevesearch/bleve/index/scorch/introducer.go

@@ -0,0 +1,449 @@
+//  Copyright (c) 2017 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package scorch
+
+import (
+	"fmt"
+	"sync/atomic"
+
+	"github.com/RoaringBitmap/roaring"
+	"github.com/blevesearch/bleve/index"
+	"github.com/blevesearch/bleve/index/scorch/segment"
+)
+
+type segmentIntroduction struct {
+	id        uint64
+	data      segment.Segment
+	obsoletes map[uint64]*roaring.Bitmap
+	ids       []string
+	internal  map[string][]byte
+
+	applied           chan error
+	persisted         chan error
+	persistedCallback index.BatchCallback
+}
+
+type persistIntroduction struct {
+	persisted map[uint64]segment.Segment
+	applied   notificationChan
+}
+
+type epochWatcher struct {
+	epoch    uint64
+	notifyCh notificationChan
+}
+
+type snapshotReversion struct {
+	snapshot  *IndexSnapshot
+	applied   chan error
+	persisted chan error
+}
+
+func (s *Scorch) mainLoop() {
+	var epochWatchers []*epochWatcher
+OUTER:
+	for {
+		atomic.AddUint64(&s.stats.TotIntroduceLoop, 1)
+
+		select {
+		case <-s.closeCh:
+			break OUTER
+
+		case epochWatcher := <-s.introducerNotifier:
+			epochWatchers = append(epochWatchers, epochWatcher)
+
+		case nextMerge := <-s.merges:
+			s.introduceMerge(nextMerge)
+
+		case next := <-s.introductions:
+			err := s.introduceSegment(next)
+			if err != nil {
+				continue OUTER
+			}
+
+		case persist := <-s.persists:
+			s.introducePersist(persist)
+
+		}
+
+		var epochCurr uint64
+		s.rootLock.RLock()
+		if s.root != nil {
+			epochCurr = s.root.epoch
+		}
+		s.rootLock.RUnlock()
+		var epochWatchersNext []*epochWatcher
+		for _, w := range epochWatchers {
+			if w.epoch < epochCurr {
+				close(w.notifyCh)
+			} else {
+				epochWatchersNext = append(epochWatchersNext, w)
+			}
+		}
+		epochWatchers = epochWatchersNext
+	}
+
+	s.asyncTasks.Done()
+}
+
+func (s *Scorch) introduceSegment(next *segmentIntroduction) error {
+	atomic.AddUint64(&s.stats.TotIntroduceSegmentBeg, 1)
+	defer atomic.AddUint64(&s.stats.TotIntroduceSegmentEnd, 1)
+
+	s.rootLock.RLock()
+	root := s.root
+	root.AddRef()
+	s.rootLock.RUnlock()
+
+	defer func() { _ = root.DecRef() }()
+
+	nsegs := len(root.segment)
+
+	// prepare new index snapshot
+	newSnapshot := &IndexSnapshot{
+		parent:   s,
+		segment:  make([]*SegmentSnapshot, 0, nsegs+1),
+		offsets:  make([]uint64, 0, nsegs+1),
+		internal: make(map[string][]byte, len(root.internal)),
+		refs:     1,
+		creator:  "introduceSegment",
+	}
+
+	// iterate through current segments
+	var running uint64
+	var docsToPersistCount, memSegments, fileSegments uint64
+	for i := range root.segment {
+		// see if optimistic work included this segment
+		delta, ok := next.obsoletes[root.segment[i].id]
+		if !ok {
+			var err error
+			delta, err = root.segment[i].segment.DocNumbers(next.ids)
+			if err != nil {
+				next.applied <- fmt.Errorf("error computing doc numbers: %v", err)
+				close(next.applied)
+				_ = newSnapshot.DecRef()
+				return err
+			}
+		}
+
+		newss := &SegmentSnapshot{
+			id:         root.segment[i].id,
+			segment:    root.segment[i].segment,
+			cachedDocs: root.segment[i].cachedDocs,
+			creator:    root.segment[i].creator,
+		}
+
+		// apply new obsoletions
+		if root.segment[i].deleted == nil {
+			newss.deleted = delta
+		} else {
+			newss.deleted = roaring.Or(root.segment[i].deleted, delta)
+		}
+		if newss.deleted.IsEmpty() {
+			newss.deleted = nil
+		}
+
+		// check for live size before copying
+		if newss.LiveSize() > 0 {
+			newSnapshot.segment = append(newSnapshot.segment, newss)
+			root.segment[i].segment.AddRef()
+			newSnapshot.offsets = append(newSnapshot.offsets, running)
+			running += newss.segment.Count()
+		}
+
+		if isMemorySegment(root.segment[i]) {
+			docsToPersistCount += root.segment[i].Count()
+			memSegments++
+		} else {
+			fileSegments++
+		}
+	}
+
+	atomic.StoreUint64(&s.stats.TotItemsToPersist, docsToPersistCount)
+	atomic.StoreUint64(&s.stats.TotMemorySegmentsAtRoot, memSegments)
+	atomic.StoreUint64(&s.stats.TotFileSegmentsAtRoot, fileSegments)
+
+	// append new segment, if any, to end of the new index snapshot
+	if next.data != nil {
+		newSegmentSnapshot := &SegmentSnapshot{
+			id:         next.id,
+			segment:    next.data, // take ownership of next.data's ref-count
+			cachedDocs: &cachedDocs{cache: nil},
+			creator:    "introduceSegment",
+		}
+		newSnapshot.segment = append(newSnapshot.segment, newSegmentSnapshot)
+		newSnapshot.offsets = append(newSnapshot.offsets, running)
+
+		// increment numItemsIntroduced which tracks the number of items
+		// queued for persistence.
+		atomic.AddUint64(&s.stats.TotIntroducedItems, newSegmentSnapshot.Count())
+		atomic.AddUint64(&s.stats.TotIntroducedSegmentsBatch, 1)
+	}
+	// copy old values
+	for key, oldVal := range root.internal {
+		newSnapshot.internal[key] = oldVal
+	}
+	// set new values and apply deletes
+	for key, newVal := range next.internal {
+		if newVal != nil {
+			newSnapshot.internal[key] = newVal
+		} else {
+			delete(newSnapshot.internal, key)
+		}
+	}
+
+	newSnapshot.updateSize()
+	s.rootLock.Lock()
+	if next.persisted != nil {
+		s.rootPersisted = append(s.rootPersisted, next.persisted)
+	}
+	if next.persistedCallback != nil {
+		s.persistedCallbacks = append(s.persistedCallbacks, next.persistedCallback)
+	}
+	// swap in new index snapshot
+	newSnapshot.epoch = s.nextSnapshotEpoch
+	s.nextSnapshotEpoch++
+	rootPrev := s.root
+	s.root = newSnapshot
+	atomic.StoreUint64(&s.stats.CurRootEpoch, s.root.epoch)
+	// release lock
+	s.rootLock.Unlock()
+
+	if rootPrev != nil {
+		_ = rootPrev.DecRef()
+	}
+
+	close(next.applied)
+
+	return nil
+}
+
+func (s *Scorch) introducePersist(persist *persistIntroduction) {
+	atomic.AddUint64(&s.stats.TotIntroducePersistBeg, 1)
+	defer atomic.AddUint64(&s.stats.TotIntroducePersistEnd, 1)
+
+	s.rootLock.Lock()
+	root := s.root
+	root.AddRef()
+	nextSnapshotEpoch := s.nextSnapshotEpoch
+	s.nextSnapshotEpoch++
+	s.rootLock.Unlock()
+
+	defer func() { _ = root.DecRef() }()
+
+	newIndexSnapshot := &IndexSnapshot{
+		parent:   s,
+		epoch:    nextSnapshotEpoch,
+		segment:  make([]*SegmentSnapshot, len(root.segment)),
+		offsets:  make([]uint64, len(root.offsets)),
+		internal: make(map[string][]byte, len(root.internal)),
+		refs:     1,
+		creator:  "introducePersist",
+	}
+
+	var docsToPersistCount, memSegments, fileSegments uint64
+	for i, segmentSnapshot := range root.segment {
+		// see if this segment has been replaced
+		if replacement, ok := persist.persisted[segmentSnapshot.id]; ok {
+			newSegmentSnapshot := &SegmentSnapshot{
+				id:         segmentSnapshot.id,
+				segment:    replacement,
+				deleted:    segmentSnapshot.deleted,
+				cachedDocs: segmentSnapshot.cachedDocs,
+				creator:    "introducePersist",
+			}
+			newIndexSnapshot.segment[i] = newSegmentSnapshot
+			delete(persist.persisted, segmentSnapshot.id)
+
+			// update items persisted incase of a new segment snapshot
+			atomic.AddUint64(&s.stats.TotPersistedItems, newSegmentSnapshot.Count())
+			atomic.AddUint64(&s.stats.TotPersistedSegments, 1)
+			fileSegments++
+		} else {
+			newIndexSnapshot.segment[i] = root.segment[i]
+			newIndexSnapshot.segment[i].segment.AddRef()
+
+			if isMemorySegment(root.segment[i]) {
+				docsToPersistCount += root.segment[i].Count()
+				memSegments++
+			} else {
+				fileSegments++
+			}
+		}
+		newIndexSnapshot.offsets[i] = root.offsets[i]
+	}
+
+	for k, v := range root.internal {
+		newIndexSnapshot.internal[k] = v
+	}
+
+	atomic.StoreUint64(&s.stats.TotItemsToPersist, docsToPersistCount)
+	atomic.StoreUint64(&s.stats.TotMemorySegmentsAtRoot, memSegments)
+	atomic.StoreUint64(&s.stats.TotFileSegmentsAtRoot, fileSegments)
+	newIndexSnapshot.updateSize()
+	s.rootLock.Lock()
+	rootPrev := s.root
+	s.root = newIndexSnapshot
+	atomic.StoreUint64(&s.stats.CurRootEpoch, s.root.epoch)
+	s.rootLock.Unlock()
+
+	if rootPrev != nil {
+		_ = rootPrev.DecRef()
+	}
+
+	close(persist.applied)
+}
+
+// The introducer should definitely handle the segmentMerge.notify
+// channel before exiting the introduceMerge.
+func (s *Scorch) introduceMerge(nextMerge *segmentMerge) {
+	atomic.AddUint64(&s.stats.TotIntroduceMergeBeg, 1)
+	defer atomic.AddUint64(&s.stats.TotIntroduceMergeEnd, 1)
+
+	s.rootLock.RLock()
+	root := s.root
+	root.AddRef()
+	s.rootLock.RUnlock()
+
+	defer func() { _ = root.DecRef() }()
+
+	newSnapshot := &IndexSnapshot{
+		parent:   s,
+		internal: root.internal,
+		refs:     1,
+		creator:  "introduceMerge",
+	}
+
+	// iterate through current segments
+	newSegmentDeleted := roaring.NewBitmap()
+	var running, docsToPersistCount, memSegments, fileSegments uint64
+	for i := range root.segment {
+		segmentID := root.segment[i].id
+		if segSnapAtMerge, ok := nextMerge.old[segmentID]; ok {
+			// this segment is going away, see if anything else was deleted since we started the merge
+			if segSnapAtMerge != nil && root.segment[i].deleted != nil {
+				// assume all these deletes are new
+				deletedSince := root.segment[i].deleted
+				// if we already knew about some of them, remove
+				if segSnapAtMerge.deleted != nil {
+					deletedSince = roaring.AndNot(root.segment[i].deleted, segSnapAtMerge.deleted)
+				}
+				deletedSinceItr := deletedSince.Iterator()
+				for deletedSinceItr.HasNext() {
+					oldDocNum := deletedSinceItr.Next()
+					newDocNum := nextMerge.oldNewDocNums[segmentID][oldDocNum]
+					newSegmentDeleted.Add(uint32(newDocNum))
+				}
+			}
+			// clean up the old segment map to figure out the
+			// obsolete segments wrt root in meantime, whatever
+			// segments left behind in old map after processing
+			// the root segments would be the obsolete segment set
+			delete(nextMerge.old, segmentID)
+		} else if root.segment[i].LiveSize() > 0 {
+			// this segment is staying
+			newSnapshot.segment = append(newSnapshot.segment, &SegmentSnapshot{
+				id:         root.segment[i].id,
+				segment:    root.segment[i].segment,
+				deleted:    root.segment[i].deleted,
+				cachedDocs: root.segment[i].cachedDocs,
+				creator:    root.segment[i].creator,
+			})
+			root.segment[i].segment.AddRef()
+			newSnapshot.offsets = append(newSnapshot.offsets, running)
+			running += root.segment[i].segment.Count()
+
+			if isMemorySegment(root.segment[i]) {
+				docsToPersistCount += root.segment[i].Count()
+				memSegments++
+			} else {
+				fileSegments++
+			}
+		}
+
+	}
+
+	// before the newMerge introduction, need to clean the newly
+	// merged segment wrt the current root segments, hence
+	// applying the obsolete segment contents to newly merged segment
+	for segID, ss := range nextMerge.old {
+		obsoleted := ss.DocNumbersLive()
+		if obsoleted != nil {
+			obsoletedIter := obsoleted.Iterator()
+			for obsoletedIter.HasNext() {
+				oldDocNum := obsoletedIter.Next()
+				newDocNum := nextMerge.oldNewDocNums[segID][oldDocNum]
+				newSegmentDeleted.Add(uint32(newDocNum))
+			}
+		}
+	}
+	// In case where all the docs in the newly merged segment getting
+	// deleted by the time we reach here, can skip the introduction.
+	if nextMerge.new != nil &&
+		nextMerge.new.Count() > newSegmentDeleted.GetCardinality() {
+		// put new segment at end
+		newSnapshot.segment = append(newSnapshot.segment, &SegmentSnapshot{
+			id:         nextMerge.id,
+			segment:    nextMerge.new, // take ownership for nextMerge.new's ref-count
+			deleted:    newSegmentDeleted,
+			cachedDocs: &cachedDocs{cache: nil},
+			creator:    "introduceMerge",
+		})
+		newSnapshot.offsets = append(newSnapshot.offsets, running)
+		atomic.AddUint64(&s.stats.TotIntroducedSegmentsMerge, 1)
+
+		switch nextMerge.new.(type) {
+		case segment.PersistedSegment:
+			fileSegments++
+		default:
+			docsToPersistCount += nextMerge.new.Count() - newSegmentDeleted.GetCardinality()
+			memSegments++
+		}
+	}
+
+	atomic.StoreUint64(&s.stats.TotItemsToPersist, docsToPersistCount)
+	atomic.StoreUint64(&s.stats.TotMemorySegmentsAtRoot, memSegments)
+	atomic.StoreUint64(&s.stats.TotFileSegmentsAtRoot, fileSegments)
+
+	newSnapshot.AddRef() // 1 ref for the nextMerge.notify response
+
+	newSnapshot.updateSize()
+	s.rootLock.Lock()
+	// swap in new index snapshot
+	newSnapshot.epoch = s.nextSnapshotEpoch
+	s.nextSnapshotEpoch++
+	rootPrev := s.root
+	s.root = newSnapshot
+	atomic.StoreUint64(&s.stats.CurRootEpoch, s.root.epoch)
+	// release lock
+	s.rootLock.Unlock()
+
+	if rootPrev != nil {
+		_ = rootPrev.DecRef()
+	}
+
+	// notify requester that we incorporated this
+	nextMerge.notify <- newSnapshot
+	close(nextMerge.notify)
+}
+
+func isMemorySegment(s *SegmentSnapshot) bool {
+	switch s.segment.(type) {
+	case segment.PersistedSegment:
+		return false
+	default:
+		return true
+	}
+}

vendor/github.com/blevesearch/bleve/index/scorch/merge.go

@@ -0,0 +1,364 @@
+//  Copyright (c) 2017 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package scorch
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"github.com/RoaringBitmap/roaring"
+	"github.com/blevesearch/bleve/index/scorch/mergeplan"
+	"github.com/blevesearch/bleve/index/scorch/segment"
+)
+
+func (s *Scorch) mergerLoop() {
+	var lastEpochMergePlanned uint64
+	mergePlannerOptions, err := s.parseMergePlannerOptions()
+	if err != nil {
+		s.fireAsyncError(fmt.Errorf("mergePlannerOption json parsing err: %v", err))
+		s.asyncTasks.Done()
+		return
+	}
+
+OUTER:
+	for {
+		atomic.AddUint64(&s.stats.TotFileMergeLoopBeg, 1)
+
+		select {
+		case <-s.closeCh:
+			break OUTER
+
+		default:
+			// check to see if there is a new snapshot to persist
+			s.rootLock.Lock()
+			ourSnapshot := s.root
+			ourSnapshot.AddRef()
+			atomic.StoreUint64(&s.iStats.mergeSnapshotSize, uint64(ourSnapshot.Size()))
+			atomic.StoreUint64(&s.iStats.mergeEpoch, ourSnapshot.epoch)
+			s.rootLock.Unlock()
+
+			if ourSnapshot.epoch != lastEpochMergePlanned {
+				startTime := time.Now()
+
+				// lets get started
+				err := s.planMergeAtSnapshot(ourSnapshot, mergePlannerOptions)
+				if err != nil {
+					atomic.StoreUint64(&s.iStats.mergeEpoch, 0)
+					if err == segment.ErrClosed {
+						// index has been closed
+						_ = ourSnapshot.DecRef()
+						break OUTER
+					}
+					s.fireAsyncError(fmt.Errorf("merging err: %v", err))
+					_ = ourSnapshot.DecRef()
+					atomic.AddUint64(&s.stats.TotFileMergeLoopErr, 1)
+					continue OUTER
+				}
+				lastEpochMergePlanned = ourSnapshot.epoch
+
+				atomic.StoreUint64(&s.stats.LastMergedEpoch, ourSnapshot.epoch)
+
+				s.fireEvent(EventKindMergerProgress, time.Since(startTime))
+			}
+			_ = ourSnapshot.DecRef()
+
+			// tell the persister we're waiting for changes
+			// first make a epochWatcher chan
+			ew := &epochWatcher{
+				epoch:    lastEpochMergePlanned,
+				notifyCh: make(notificationChan, 1),
+			}
+
+			// give it to the persister
+			select {
+			case <-s.closeCh:
+				break OUTER
+			case s.persisterNotifier <- ew:
+			}
+
+			// now wait for persister (but also detect close)
+			select {
+			case <-s.closeCh:
+				break OUTER
+			case <-ew.notifyCh:
+			}
+		}
+
+		atomic.AddUint64(&s.stats.TotFileMergeLoopEnd, 1)
+	}
+
+	s.asyncTasks.Done()
+}
+
+func (s *Scorch) parseMergePlannerOptions() (*mergeplan.MergePlanOptions,
+	error) {
+	mergePlannerOptions := mergeplan.DefaultMergePlanOptions
+	if v, ok := s.config["scorchMergePlanOptions"]; ok {
+		b, err := json.Marshal(v)
+		if err != nil {
+			return &mergePlannerOptions, err
+		}
+
+		err = json.Unmarshal(b, &mergePlannerOptions)
+		if err != nil {
+			return &mergePlannerOptions, err
+		}
+
+		err = mergeplan.ValidateMergePlannerOptions(&mergePlannerOptions)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return &mergePlannerOptions, nil
+}
+
+func (s *Scorch) planMergeAtSnapshot(ourSnapshot *IndexSnapshot,
+	options *mergeplan.MergePlanOptions) error {
+	// build list of persisted segments in this snapshot
+	var onlyPersistedSnapshots []mergeplan.Segment
+	for _, segmentSnapshot := range ourSnapshot.segment {
+		if _, ok := segmentSnapshot.segment.(segment.PersistedSegment); ok {
+			onlyPersistedSnapshots = append(onlyPersistedSnapshots, segmentSnapshot)
+		}
+	}
+
+	atomic.AddUint64(&s.stats.TotFileMergePlan, 1)
+
+	// give this list to the planner
+	resultMergePlan, err := mergeplan.Plan(onlyPersistedSnapshots, options)
+	if err != nil {
+		atomic.AddUint64(&s.stats.TotFileMergePlanErr, 1)
+		return fmt.Errorf("merge planning err: %v", err)
+	}
+	if resultMergePlan == nil {
+		// nothing to do
+		atomic.AddUint64(&s.stats.TotFileMergePlanNone, 1)
+		return nil
+	}
+	atomic.AddUint64(&s.stats.TotFileMergePlanOk, 1)
+
+	atomic.AddUint64(&s.stats.TotFileMergePlanTasks, uint64(len(resultMergePlan.Tasks)))
+
+	// process tasks in serial for now
+	var filenames []string
+
+	for _, task := range resultMergePlan.Tasks {
+		if len(task.Segments) == 0 {
+			atomic.AddUint64(&s.stats.TotFileMergePlanTasksSegmentsEmpty, 1)
+			continue
+		}
+
+		atomic.AddUint64(&s.stats.TotFileMergePlanTasksSegments, uint64(len(task.Segments)))
+
+		oldMap := make(map[uint64]*SegmentSnapshot)
+		newSegmentID := atomic.AddUint64(&s.nextSegmentID, 1)
+		segmentsToMerge := make([]segment.Segment, 0, len(task.Segments))
+		docsToDrop := make([]*roaring.Bitmap, 0, len(task.Segments))
+
+		for _, planSegment := range task.Segments {
+			if segSnapshot, ok := planSegment.(*SegmentSnapshot); ok {
+				oldMap[segSnapshot.id] = segSnapshot
+				if persistedSeg, ok := segSnapshot.segment.(segment.PersistedSegment); ok {
+					if segSnapshot.LiveSize() == 0 {
+						atomic.AddUint64(&s.stats.TotFileMergeSegmentsEmpty, 1)
+						oldMap[segSnapshot.id] = nil
+					} else {
+						segmentsToMerge = append(segmentsToMerge, segSnapshot.segment)
+						docsToDrop = append(docsToDrop, segSnapshot.deleted)
+					}
+					// track the files getting merged for unsetting the
+					// removal ineligibility. This helps to unflip files
+					// even with fast merger, slow persister work flows.
+					path := persistedSeg.Path()
+					filenames = append(filenames,
+						strings.TrimPrefix(path, s.path+string(os.PathSeparator)))
+				}
+			}
+		}
+
+		var oldNewDocNums map[uint64][]uint64
+		var seg segment.Segment
+		if len(segmentsToMerge) > 0 {
+			filename := zapFileName(newSegmentID)
+			s.markIneligibleForRemoval(filename)
+			path := s.path + string(os.PathSeparator) + filename
+
+			fileMergeZapStartTime := time.Now()
+
+			atomic.AddUint64(&s.stats.TotFileMergeZapBeg, 1)
+			newDocNums, _, err := s.segPlugin.Merge(segmentsToMerge, docsToDrop, path,
+				s.closeCh, s)
+			atomic.AddUint64(&s.stats.TotFileMergeZapEnd, 1)
+
+			fileMergeZapTime := uint64(time.Since(fileMergeZapStartTime))
+			atomic.AddUint64(&s.stats.TotFileMergeZapTime, fileMergeZapTime)
+			if atomic.LoadUint64(&s.stats.MaxFileMergeZapTime) < fileMergeZapTime {
+				atomic.StoreUint64(&s.stats.MaxFileMergeZapTime, fileMergeZapTime)
+			}
+
+			if err != nil {
+				s.unmarkIneligibleForRemoval(filename)
+				atomic.AddUint64(&s.stats.TotFileMergePlanTasksErr, 1)
+				if err == segment.ErrClosed {
+					return err
+				}
+				return fmt.Errorf("merging failed: %v", err)
+			}
+
+			seg, err = s.segPlugin.Open(path)
+			if err != nil {
+				s.unmarkIneligibleForRemoval(filename)
+				atomic.AddUint64(&s.stats.TotFileMergePlanTasksErr, 1)
+				return err
+			}
+			oldNewDocNums = make(map[uint64][]uint64)
+			for i, segNewDocNums := range newDocNums {
+				oldNewDocNums[task.Segments[i].Id()] = segNewDocNums
+			}
+
+			atomic.AddUint64(&s.stats.TotFileMergeSegments, uint64(len(segmentsToMerge)))
+		}
+
+		sm := &segmentMerge{
+			id:            newSegmentID,
+			old:           oldMap,
+			oldNewDocNums: oldNewDocNums,
+			new:           seg,
+			notify:        make(chan *IndexSnapshot),
+		}
+
+		// give it to the introducer
+		select {
+		case <-s.closeCh:
+			_ = seg.Close()
+			return segment.ErrClosed
+		case s.merges <- sm:
+			atomic.AddUint64(&s.stats.TotFileMergeIntroductions, 1)
+		}
+
+		introStartTime := time.Now()
+		// it is safe to blockingly wait for the merge introduction
+		// here as the introducer is bound to handle the notify channel.
+		newSnapshot := <-sm.notify
+		introTime := uint64(time.Since(introStartTime))
+		atomic.AddUint64(&s.stats.TotFileMergeZapIntroductionTime, introTime)
+		if atomic.LoadUint64(&s.stats.MaxFileMergeZapIntroductionTime) < introTime {
+			atomic.StoreUint64(&s.stats.MaxFileMergeZapIntroductionTime, introTime)
+		}
+		atomic.AddUint64(&s.stats.TotFileMergeIntroductionsDone, 1)
+		if newSnapshot != nil {
+			_ = newSnapshot.DecRef()
+		}
+
+		atomic.AddUint64(&s.stats.TotFileMergePlanTasksDone, 1)
+	}
+
+	// once all the newly merged segment introductions are done,
+	// its safe to unflip the removal ineligibility for the replaced
+	// older segments
+	for _, f := range filenames {
+		s.unmarkIneligibleForRemoval(f)
+	}
+
+	return nil
+}
+
+type segmentMerge struct {
+	id            uint64
+	old           map[uint64]*SegmentSnapshot
+	oldNewDocNums map[uint64][]uint64
+	new           segment.Segment
+	notify        chan *IndexSnapshot
+}
+
+// perform a merging of the given SegmentBase instances into a new,
+// persisted segment, and synchronously introduce that new segment
+// into the root
+func (s *Scorch) mergeSegmentBases(snapshot *IndexSnapshot,
+	sbs []segment.Segment, sbsDrops []*roaring.Bitmap,
+	sbsIndexes []int) (*IndexSnapshot, uint64, error) {
+	atomic.AddUint64(&s.stats.TotMemMergeBeg, 1)
+
+	memMergeZapStartTime := time.Now()
+
+	atomic.AddUint64(&s.stats.TotMemMergeZapBeg, 1)
+
+	newSegmentID := atomic.AddUint64(&s.nextSegmentID, 1)
+	filename := zapFileName(newSegmentID)
+	path := s.path + string(os.PathSeparator) + filename
+
+	newDocNums, _, err :=
+		s.segPlugin.Merge(sbs, sbsDrops, path, s.closeCh, s)
+
+	atomic.AddUint64(&s.stats.TotMemMergeZapEnd, 1)
+
+	memMergeZapTime := uint64(time.Since(memMergeZapStartTime))
+	atomic.AddUint64(&s.stats.TotMemMergeZapTime, memMergeZapTime)
+	if atomic.LoadUint64(&s.stats.MaxMemMergeZapTime) < memMergeZapTime {
+		atomic.StoreUint64(&s.stats.MaxMemMergeZapTime, memMergeZapTime)
+	}
+
+	if err != nil {
+		atomic.AddUint64(&s.stats.TotMemMergeErr, 1)
+		return nil, 0, err
+	}
+
+	seg, err := s.segPlugin.Open(path)
+	if err != nil {
+		atomic.AddUint64(&s.stats.TotMemMergeErr, 1)
+		return nil, 0, err
+	}
+
+	// update persisted stats
+	atomic.AddUint64(&s.stats.TotPersistedItems, seg.Count())
+	atomic.AddUint64(&s.stats.TotPersistedSegments, 1)
+
+	sm := &segmentMerge{
+		id:            newSegmentID,
+		old:           make(map[uint64]*SegmentSnapshot),
+		oldNewDocNums: make(map[uint64][]uint64),
+		new:           seg,
+		notify:        make(chan *IndexSnapshot),
+	}
+
+	for i, idx := range sbsIndexes {
+		ss := snapshot.segment[idx]
+		sm.old[ss.id] = ss
+		sm.oldNewDocNums[ss.id] = newDocNums[i]
+	}
+
+	select { // send to introducer
+	case <-s.closeCh:
+		_ = seg.DecRef()
+		return nil, 0, segment.ErrClosed
+	case s.merges <- sm:
+	}
+
+	// blockingly wait for the introduction to complete
+	newSnapshot := <-sm.notify
+	if newSnapshot != nil {
+		atomic.AddUint64(&s.stats.TotMemMergeSegments, uint64(len(sbs)))
+		atomic.AddUint64(&s.stats.TotMemMergeDone, 1)
+	}
+	return newSnapshot, newSegmentID, nil
+}
+
+func (s *Scorch) ReportBytesWritten(bytesWritten uint64) {
+	atomic.AddUint64(&s.stats.TotFileMergeWrittenBytes, bytesWritten)
+}

vendor/github.com/blevesearch/bleve/index/scorch/mergeplan/merge_plan.go

@@ -0,0 +1,386 @@
+//  Copyright (c) 2017 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package mergeplan provides a segment merge planning approach that's
+// inspired by Lucene's TieredMergePolicy.java and descriptions like
+// http://blog.mikemccandless.com/2011/02/visualizing-lucenes-segment-merges.html
+package mergeplan
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"sort"
+	"strings"
+)
+
+// A Segment represents the information that the planner needs to
+// calculate segment merging.
+type Segment interface {
+	// Unique id of the segment -- used for sorting.
+	Id() uint64
+
+	// Full segment size (the size before any logical deletions).
+	FullSize() int64
+
+	// Size of the live data of the segment; i.e., FullSize() minus
+	// any logical deletions.
+	LiveSize() int64
+}
+
+// Plan() will functionally compute a merge plan.  A segment will be
+// assigned to at most a single MergeTask in the output MergePlan.  A
+// segment not assigned to any MergeTask means the segment should
+// remain unmerged.
+func Plan(segments []Segment, o *MergePlanOptions) (*MergePlan, error) {
+	return plan(segments, o)
+}
+
+// A MergePlan is the result of the Plan() API.
+//
+// The planner doesn’t know how or whether these tasks are executed --
+// that’s up to a separate merge execution system, which might execute
+// these tasks concurrently or not, and which might execute all the
+// tasks or not.
+type MergePlan struct {
+	Tasks []*MergeTask
+}
+
+// A MergeTask represents several segments that should be merged
+// together into a single segment.
+type MergeTask struct {
+	Segments []Segment
+}
+
+// The MergePlanOptions is designed to be reusable between planning calls.
+type MergePlanOptions struct {
+	// Max # segments per logarithmic tier, or max width of any
+	// logarithmic “step”.  Smaller values mean more merging but fewer
+	// segments.  Should be >= SegmentsPerMergeTask, else you'll have
+	// too much merging.
+	MaxSegmentsPerTier int
+
+	// Max size of any segment produced after merging.  Actual
+	// merging, however, may produce segment sizes different than the
+	// planner’s predicted sizes.
+	MaxSegmentSize int64
+
+	// The growth factor for each tier in a staircase of idealized
+	// segments computed by CalcBudget().
+	TierGrowth float64
+
+	// The number of segments in any resulting MergeTask.  e.g.,
+	// len(result.Tasks[ * ].Segments) == SegmentsPerMergeTask.
+	SegmentsPerMergeTask int
+
+	// Small segments are rounded up to this size, i.e., treated as
+	// equal (floor) size for consideration.  This is to prevent lots
+	// of tiny segments from resulting in a long tail in the index.
+	FloorSegmentSize int64
+
+	// Controls how aggressively merges that reclaim more deletions
+	// are favored.  Higher values will more aggressively target
+	// merges that reclaim deletions, but be careful not to go so high
+	// that way too much merging takes place; a value of 3.0 is
+	// probably nearly too high.  A value of 0.0 means deletions don't
+	// impact merge selection.
+	ReclaimDeletesWeight float64
+
+	// Optional, defaults to mergeplan.CalcBudget().
+	CalcBudget func(totalSize int64, firstTierSize int64,
+		o *MergePlanOptions) (budgetNumSegments int)
+
+	// Optional, defaults to mergeplan.ScoreSegments().
+	ScoreSegments func(segments []Segment, o *MergePlanOptions) float64
+
+	// Optional.
+	Logger func(string)
+}
+
+// Returns the higher of the input or FloorSegmentSize.
+func (o *MergePlanOptions) RaiseToFloorSegmentSize(s int64) int64 {
+	if s > o.FloorSegmentSize {
+		return s
+	}
+	return o.FloorSegmentSize
+}
+
+// MaxSegmentSizeLimit represents the maximum size of a segment,
+// this limit comes with hit-1 optimisation/max encoding limit uint31.
+const MaxSegmentSizeLimit = 1<<31 - 1
+
+// ErrMaxSegmentSizeTooLarge is returned when the size of the segment
+// exceeds the MaxSegmentSizeLimit
+var ErrMaxSegmentSizeTooLarge = errors.New("MaxSegmentSize exceeds the size limit")
+
+// DefaultMergePlanOptions suggests the default options.
+var DefaultMergePlanOptions = MergePlanOptions{
+	MaxSegmentsPerTier:   10,
+	MaxSegmentSize:       5000000,
+	TierGrowth:           10.0,
+	SegmentsPerMergeTask: 10,
+	FloorSegmentSize:     2000,
+	ReclaimDeletesWeight: 2.0,
+}
+
+// -------------------------------------------
+
+func plan(segmentsIn []Segment, o *MergePlanOptions) (*MergePlan, error) {
+	if len(segmentsIn) <= 1 {
+		return nil, nil
+	}
+
+	if o == nil {
+		o = &DefaultMergePlanOptions
+	}
+
+	segments := append([]Segment(nil), segmentsIn...) // Copy.
+
+	sort.Sort(byLiveSizeDescending(segments))
+
+	var minLiveSize int64 = math.MaxInt64
+
+	var eligibles []Segment
+	var eligiblesLiveSize int64
+
+	for _, segment := range segments {
+		if minLiveSize > segment.LiveSize() {
+			minLiveSize = segment.LiveSize()
+		}
+
+		// Only small-enough segments are eligible.
+		if segment.LiveSize() < o.MaxSegmentSize/2 {
+			eligibles = append(eligibles, segment)
+			eligiblesLiveSize += segment.LiveSize()
+		}
+	}
+
+	minLiveSize = o.RaiseToFloorSegmentSize(minLiveSize)
+
+	calcBudget := o.CalcBudget
+	if calcBudget == nil {
+		calcBudget = CalcBudget
+	}
+
+	budgetNumSegments := CalcBudget(eligiblesLiveSize, minLiveSize, o)
+
+	scoreSegments := o.ScoreSegments
+	if scoreSegments == nil {
+		scoreSegments = ScoreSegments
+	}
+
+	rv := &MergePlan{}
+
+	var empties []Segment
+	for _, eligible := range eligibles {
+		if eligible.LiveSize() <= 0 {
+			empties = append(empties, eligible)
+		}
+	}
+	if len(empties) > 0 {
+		rv.Tasks = append(rv.Tasks, &MergeTask{Segments: empties})
+		eligibles = removeSegments(eligibles, empties)
+	}
+
+	// While we’re over budget, keep looping, which might produce
+	// another MergeTask.
+	for len(eligibles) > 0 && (len(eligibles)+len(rv.Tasks)) > budgetNumSegments {
+		// Track a current best roster as we examine and score
+		// potential rosters of merges.
+		var bestRoster []Segment
+		var bestRosterScore float64 // Lower score is better.
+
+		for startIdx := 0; startIdx < len(eligibles); startIdx++ {
+			var roster []Segment
+			var rosterLiveSize int64
+
+			for idx := startIdx; idx < len(eligibles) && len(roster) < o.SegmentsPerMergeTask; idx++ {
+				eligible := eligibles[idx]
+
+				if rosterLiveSize+eligible.LiveSize() < o.MaxSegmentSize {
+					roster = append(roster, eligible)
+					rosterLiveSize += eligible.LiveSize()
+				}
+			}
+
+			if len(roster) > 0 {
+				rosterScore := scoreSegments(roster, o)
+
+				if len(bestRoster) == 0 || rosterScore < bestRosterScore {
+					bestRoster = roster
+					bestRosterScore = rosterScore
+				}
+			}
+		}
+
+		if len(bestRoster) == 0 {
+			return rv, nil
+		}
+
+		rv.Tasks = append(rv.Tasks, &MergeTask{Segments: bestRoster})
+
+		eligibles = removeSegments(eligibles, bestRoster)
+	}
+
+	return rv, nil
+}
+
+// Compute the number of segments that would be needed to cover the
+// totalSize, by climbing up a logarithmically growing staircase of
+// segment tiers.
+func CalcBudget(totalSize int64, firstTierSize int64, o *MergePlanOptions) (
+	budgetNumSegments int) {
+	tierSize := firstTierSize
+	if tierSize < 1 {
+		tierSize = 1
+	}
+
+	maxSegmentsPerTier := o.MaxSegmentsPerTier
+	if maxSegmentsPerTier < 1 {
+		maxSegmentsPerTier = 1
+	}
+
+	tierGrowth := o.TierGrowth
+	if tierGrowth < 1.0 {
+		tierGrowth = 1.0
+	}
+
+	for totalSize > 0 {
+		segmentsInTier := float64(totalSize) / float64(tierSize)
+		if segmentsInTier < float64(maxSegmentsPerTier) {
+			budgetNumSegments += int(math.Ceil(segmentsInTier))
+			break
+		}
+
+		budgetNumSegments += maxSegmentsPerTier
+		totalSize -= int64(maxSegmentsPerTier) * tierSize
+		tierSize = int64(float64(tierSize) * tierGrowth)
+	}
+
+	return budgetNumSegments
+}
+
+// Of note, removeSegments() keeps the ordering of the results stable.
+func removeSegments(segments []Segment, toRemove []Segment) []Segment {
+	rv := make([]Segment, 0, len(segments)-len(toRemove))
+OUTER:
+	for _, segment := range segments {
+		for _, r := range toRemove {
+			if segment == r {
+				continue OUTER
+			}
+		}
+		rv = append(rv, segment)
+	}
+	return rv
+}
+
+// Smaller result score is better.
+func ScoreSegments(segments []Segment, o *MergePlanOptions) float64 {
+	var totBeforeSize int64
+	var totAfterSize int64
+	var totAfterSizeFloored int64
+
+	for _, segment := range segments {
+		totBeforeSize += segment.FullSize()
+		totAfterSize += segment.LiveSize()
+		totAfterSizeFloored += o.RaiseToFloorSegmentSize(segment.LiveSize())
+	}
+
+	if totBeforeSize <= 0 || totAfterSize <= 0 || totAfterSizeFloored <= 0 {
+		return 0
+	}
+
+	// Roughly guess the "balance" of the segments -- whether the
+	// segments are about the same size.
+	balance :=
+		float64(o.RaiseToFloorSegmentSize(segments[0].LiveSize())) /
+			float64(totAfterSizeFloored)
+
+	// Gently favor smaller merges over bigger ones.  We don't want to
+	// make the exponent too large else we end up with poor merges of
+	// small segments in order to avoid the large merges.
+	score := balance * math.Pow(float64(totAfterSize), 0.05)
+
+	// Strongly favor merges that reclaim deletes.
+	nonDelRatio := float64(totAfterSize) / float64(totBeforeSize)
+
+	score *= math.Pow(nonDelRatio, o.ReclaimDeletesWeight)
+
+	return score
+}
+
+// ------------------------------------------
+
+// ToBarChart returns an ASCII rendering of the segments and the plan.
+// The barMax is the max width of the bars in the bar chart.
+func ToBarChart(prefix string, barMax int, segments []Segment, plan *MergePlan) string {
+	rv := make([]string, 0, len(segments))
+
+	var maxFullSize int64
+	for _, segment := range segments {
+		if maxFullSize < segment.FullSize() {
+			maxFullSize = segment.FullSize()
+		}
+	}
+	if maxFullSize < 0 {
+		maxFullSize = 1
+	}
+
+	for _, segment := range segments {
+		barFull := int(segment.FullSize())
+		barLive := int(segment.LiveSize())
+
+		if maxFullSize > int64(barMax) {
+			barFull = int(float64(barMax) * float64(barFull) / float64(maxFullSize))
+			barLive = int(float64(barMax) * float64(barLive) / float64(maxFullSize))
+		}
+
+		barKind := " "
+		barChar := "."
+
+		if plan != nil {
+		TASK_LOOP:
+			for taski, task := range plan.Tasks {
+				for _, taskSegment := range task.Segments {
+					if taskSegment == segment {
+						barKind = "*"
+						barChar = fmt.Sprintf("%d", taski)
+						break TASK_LOOP
+					}
+				}
+			}
+		}
+
+		bar :=
+			strings.Repeat(barChar, barLive)[0:barLive] +
+				strings.Repeat("x", barFull-barLive)[0:barFull-barLive]
+
+		rv = append(rv, fmt.Sprintf("%s %5d: %5d /%5d - %s %s", prefix,
+			segment.Id(),
+			segment.LiveSize(),
+			segment.FullSize(),
+			barKind, bar))
+	}
+
+	return strings.Join(rv, "\n")
+}
+
+// ValidateMergePlannerOptions validates the merge planner options
+func ValidateMergePlannerOptions(options *MergePlanOptions) error {
+	if options.MaxSegmentSize > MaxSegmentSizeLimit {
+		return ErrMaxSegmentSizeTooLarge
+	}
+	return nil
+}

vendor/github.com/blevesearch/bleve/index/scorch/mergeplan/sort.go

@@ -0,0 +1,28 @@
+//  Copyright (c) 2017 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mergeplan
+
+type byLiveSizeDescending []Segment
+
+func (a byLiveSizeDescending) Len() int { return len(a) }
+
+func (a byLiveSizeDescending) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+
+func (a byLiveSizeDescending) Less(i, j int) bool {
+	if a[i].LiveSize() != a[j].LiveSize() {
+		return a[i].LiveSize() > a[j].LiveSize()
+	}
+	return a[i].Id() < a[j].Id()
+}

vendor/github.com/blevesearch/bleve/index/scorch/optimize.go

@@ -0,0 +1,396 @@
+//  Copyright (c) 2018 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package scorch
+
+import (
+	"fmt"
+
+	"github.com/RoaringBitmap/roaring"
+	"github.com/blevesearch/bleve/index"
+	"github.com/blevesearch/bleve/index/scorch/segment"
+)
+
+var OptimizeConjunction = true
+var OptimizeConjunctionUnadorned = true
+var OptimizeDisjunctionUnadorned = true
+
+func (s *IndexSnapshotTermFieldReader) Optimize(kind string,
+	octx index.OptimizableContext) (index.OptimizableContext, error) {
+	if OptimizeConjunction && kind == "conjunction" {
+		return s.optimizeConjunction(octx)
+	}
+
+	if OptimizeConjunctionUnadorned && kind == "conjunction:unadorned" {
+		return s.optimizeConjunctionUnadorned(octx)
+	}
+
+	if OptimizeDisjunctionUnadorned && kind == "disjunction:unadorned" {
+		return s.optimizeDisjunctionUnadorned(octx)
+	}
+
+	return octx, nil
+}
+
+var OptimizeDisjunctionUnadornedMinChildCardinality = uint64(256)
+
+// ----------------------------------------------------------------
+
+func (s *IndexSnapshotTermFieldReader) optimizeConjunction(
+	octx index.OptimizableContext) (index.OptimizableContext, error) {
+	if octx == nil {
+		octx = &OptimizeTFRConjunction{snapshot: s.snapshot}
+	}
+
+	o, ok := octx.(*OptimizeTFRConjunction)
+	if !ok {
+		return octx, nil
+	}
+
+	if o.snapshot != s.snapshot {
+		return nil, fmt.Errorf("tried to optimize conjunction across different snapshots")
+	}
+
+	o.tfrs = append(o.tfrs, s)
+
+	return o, nil
+}
+
+type OptimizeTFRConjunction struct {
+	snapshot *IndexSnapshot
+
+	tfrs []*IndexSnapshotTermFieldReader
+}
+
+func (o *OptimizeTFRConjunction) Finish() (index.Optimized, error) {
+	if len(o.tfrs) <= 1 {
+		return nil, nil
+	}
+
+	for i := range o.snapshot.segment {
+		itr0, ok := o.tfrs[0].iterators[i].(segment.OptimizablePostingsIterator)
+		if !ok || itr0.ActualBitmap() == nil {
+			continue
+		}
+
+		itr1, ok := o.tfrs[1].iterators[i].(segment.OptimizablePostingsIterator)
+		if !ok || itr1.ActualBitmap() == nil {
+			continue
+		}
+
+		bm := roaring.And(itr0.ActualBitmap(), itr1.ActualBitmap())
+
+		for _, tfr := range o.tfrs[2:] {
+			itr, ok := tfr.iterators[i].(segment.OptimizablePostingsIterator)
+			if !ok || itr.ActualBitmap() == nil {
+				continue
+			}
+
+			bm.And(itr.ActualBitmap())
+		}
+
+		// in this conjunction optimization, the postings iterators
+		// will all share the same AND'ed together actual bitmap.  The
+		// regular conjunction searcher machinery will still be used,
+		// but the underlying bitmap will be smaller.
+		for _, tfr := range o.tfrs {
+			itr, ok := tfr.iterators[i].(segment.OptimizablePostingsIterator)
+			if ok && itr.ActualBitmap() != nil {
+				itr.ReplaceActual(bm)
+			}
+		}
+	}
+
+	return nil, nil
+}
+
+// ----------------------------------------------------------------
+
+// An "unadorned" conjunction optimization is appropriate when
+// additional or subsidiary information like freq-norm's and
+// term-vectors are not required, and instead only the internal-id's
+// are needed.
+func (s *IndexSnapshotTermFieldReader) optimizeConjunctionUnadorned(
+	octx index.OptimizableContext) (index.OptimizableContext, error) {
+	if octx == nil {
+		octx = &OptimizeTFRConjunctionUnadorned{snapshot: s.snapshot}
+	}
+
+	o, ok := octx.(*OptimizeTFRConjunctionUnadorned)
+	if !ok {
+		return nil, nil
+	}
+
+	if o.snapshot != s.snapshot {
+		return nil, fmt.Errorf("tried to optimize unadorned conjunction across different snapshots")
+	}
+
+	o.tfrs = append(o.tfrs, s)
+
+	return o, nil
+}
+
+type OptimizeTFRConjunctionUnadorned struct {
+	snapshot *IndexSnapshot
+
+	tfrs []*IndexSnapshotTermFieldReader
+}
+
+var OptimizeTFRConjunctionUnadornedTerm = []byte("<conjunction:unadorned>")
+var OptimizeTFRConjunctionUnadornedField = "*"
+
+// Finish of an unadorned conjunction optimization will compute a
+// termFieldReader with an "actual" bitmap that represents the
+// constituent bitmaps AND'ed together.  This termFieldReader cannot
+// provide any freq-norm or termVector associated information.
+func (o *OptimizeTFRConjunctionUnadorned) Finish() (rv index.Optimized, err error) {
+	if len(o.tfrs) <= 1 {
+		return nil, nil
+	}
+
+	// We use an artificial term and field because the optimized
+	// termFieldReader can represent multiple terms and fields.
+	oTFR := &IndexSnapshotTermFieldReader{
+		term:               OptimizeTFRConjunctionUnadornedTerm,
+		field:              OptimizeTFRConjunctionUnadornedField,
+		snapshot:           o.snapshot,
+		iterators:          make([]segment.PostingsIterator, len(o.snapshot.segment)),
+		segmentOffset:      0,
+		includeFreq:        false,
+		includeNorm:        false,
+		includeTermVectors: false,
+	}
+
+	var actualBMs []*roaring.Bitmap // Collected from regular posting lists.
+
+OUTER:
+	for i := range o.snapshot.segment {
+		actualBMs = actualBMs[:0]
+
+		var docNum1HitLast uint64
+		var docNum1HitLastOk bool
+
+		for _, tfr := range o.tfrs {
+			if _, ok := tfr.iterators[i].(*segment.EmptyPostingsIterator); ok {
+				// An empty postings iterator means the entire AND is empty.
+				oTFR.iterators[i] = segment.AnEmptyPostingsIterator
+				continue OUTER
+			}
+
+			itr, ok := tfr.iterators[i].(segment.OptimizablePostingsIterator)
+			if !ok {
+				// We only optimize postings iterators that support this operation.
+				return nil, nil
+			}
+
+			// If the postings iterator is "1-hit" optimized, then we
+			// can perform several optimizations up-front here.
+			docNum1Hit, ok := itr.DocNum1Hit()
+			if ok {
+				if docNum1HitLastOk && docNum1HitLast != docNum1Hit {
+					// The docNum1Hit doesn't match the previous
+					// docNum1HitLast, so the entire AND is empty.
+					oTFR.iterators[i] = segment.AnEmptyPostingsIterator
+					continue OUTER
+				}
+
+				docNum1HitLast = docNum1Hit
+				docNum1HitLastOk = true
+
+				continue
+			}
+
+			if itr.ActualBitmap() == nil {
+				// An empty actual bitmap means the entire AND is empty.
+				oTFR.iterators[i] = segment.AnEmptyPostingsIterator
+				continue OUTER
+			}
+
+			// Collect the actual bitmap for more processing later.
+			actualBMs = append(actualBMs, itr.ActualBitmap())
+		}
+
+		if docNum1HitLastOk {
+			// We reach here if all the 1-hit optimized posting
+			// iterators had the same 1-hit docNum, so we can check if
+			// our collected actual bitmaps also have that docNum.
+			for _, bm := range actualBMs {
+				if !bm.Contains(uint32(docNum1HitLast)) {
+					// The docNum1Hit isn't in one of our actual
+					// bitmaps, so the entire AND is empty.
+					oTFR.iterators[i] = segment.AnEmptyPostingsIterator
+					continue OUTER
+				}
+			}
+
+			// The actual bitmaps and docNum1Hits all contain or have
+			// the same 1-hit docNum, so that's our AND'ed result.
+			oTFR.iterators[i] = segment.NewUnadornedPostingsIteratorFrom1Hit(docNum1HitLast)
+
+			continue OUTER
+		}
+
+		if len(actualBMs) == 0 {
+			// If we've collected no actual bitmaps at this point,
+			// then the entire AND is empty.
+			oTFR.iterators[i] = segment.AnEmptyPostingsIterator
+			continue OUTER
+		}
+
+		if len(actualBMs) == 1 {
+			// If we've only 1 actual bitmap, then that's our result.
+			oTFR.iterators[i] = segment.NewUnadornedPostingsIteratorFromBitmap(actualBMs[0])
+
+			continue OUTER
+		}
+
+		// Else, AND together our collected bitmaps as our result.
+		bm := roaring.And(actualBMs[0], actualBMs[1])
+
+		for _, actualBM := range actualBMs[2:] {
+			bm.And(actualBM)
+		}
+
+		oTFR.iterators[i] = segment.NewUnadornedPostingsIteratorFromBitmap(bm)
+	}
+
+	return oTFR, nil
+}
+
+// ----------------------------------------------------------------
+
+// An "unadorned" disjunction optimization is appropriate when
+// additional or subsidiary information like freq-norm's and
+// term-vectors are not required, and instead only the internal-id's
+// are needed.
+func (s *IndexSnapshotTermFieldReader) optimizeDisjunctionUnadorned(
+	octx index.OptimizableContext) (index.OptimizableContext, error) {
+	if octx == nil {
+		octx = &OptimizeTFRDisjunctionUnadorned{snapshot: s.snapshot}
+	}
+
+	o, ok := octx.(*OptimizeTFRDisjunctionUnadorned)
+	if !ok {
+		return nil, nil
+	}
+
+	if o.snapshot != s.snapshot {
+		return nil, fmt.Errorf("tried to optimize unadorned disjunction across different snapshots")
+	}
+
+	o.tfrs = append(o.tfrs, s)
+
+	return o, nil
+}
+
+type OptimizeTFRDisjunctionUnadorned struct {
+	snapshot *IndexSnapshot
+
+	tfrs []*IndexSnapshotTermFieldReader
+}
+
+var OptimizeTFRDisjunctionUnadornedTerm = []byte("<disjunction:unadorned>")
+var OptimizeTFRDisjunctionUnadornedField = "*"
+
+// Finish of an unadorned disjunction optimization will compute a
+// termFieldReader with an "actual" bitmap that represents the
+// constituent bitmaps OR'ed together.  This termFieldReader cannot
+// provide any freq-norm or termVector associated information.
+func (o *OptimizeTFRDisjunctionUnadorned) Finish() (rv index.Optimized, err error) {
+	if len(o.tfrs) <= 1 {
+		return nil, nil
+	}
+
+	for i := range o.snapshot.segment {
+		var cMax uint64
+
+		for _, tfr := range o.tfrs {
+			itr, ok := tfr.iterators[i].(segment.OptimizablePostingsIterator)
+			if !ok {
+				return nil, nil
+			}
+
+			if itr.ActualBitmap() != nil {
+				c := itr.ActualBitmap().GetCardinality()
+				if cMax < c {
+					cMax = c
+				}
+			}
+		}
+
+		// Heuristic to skip the optimization if all the constituent
+		// bitmaps are too small, where the processing & resource
+		// overhead to create the OR'ed bitmap outweighs the benefit.
+		if cMax < OptimizeDisjunctionUnadornedMinChildCardinality {
+			return nil, nil
+		}
+	}
+
+	// We use an artificial term and field because the optimized
+	// termFieldReader can represent multiple terms and fields.
+	oTFR := &IndexSnapshotTermFieldReader{
+		term:               OptimizeTFRDisjunctionUnadornedTerm,
+		field:              OptimizeTFRDisjunctionUnadornedField,
+		snapshot:           o.snapshot,
+		iterators:          make([]segment.PostingsIterator, len(o.snapshot.segment)),
+		segmentOffset:      0,
+		includeFreq:        false,
+		includeNorm:        false,
+		includeTermVectors: false,
+	}
+
+	var docNums []uint32            // Collected docNum's from 1-hit posting lists.
+	var actualBMs []*roaring.Bitmap // Collected from regular posting lists.
+
+	for i := range o.snapshot.segment {
+		docNums = docNums[:0]
+		actualBMs = actualBMs[:0]
+
+		for _, tfr := range o.tfrs {
+			itr, ok := tfr.iterators[i].(segment.OptimizablePostingsIterator)
+			if !ok {
+				return nil, nil
+			}
+
+			docNum, ok := itr.DocNum1Hit()
+			if ok {
+				docNums = append(docNums, uint32(docNum))
+				continue
+			}
+
+			if itr.ActualBitmap() != nil {
+				actualBMs = append(actualBMs, itr.ActualBitmap())
+			}
+		}
+
+		var bm *roaring.Bitmap
+		if len(actualBMs) > 2 {
+			bm = roaring.HeapOr(actualBMs...)
+		} else if len(actualBMs) == 2 {
+			bm = roaring.Or(actualBMs[0], actualBMs[1])
+		} else if len(actualBMs) == 1 {
+			bm = actualBMs[0].Clone()
+		}
+
+		if bm == nil {
+			bm = roaring.New()
+		}
+
+		bm.AddMany(docNums)
+
+		oTFR.iterators[i] = segment.NewUnadornedPostingsIteratorFromBitmap(bm)
+	}
+
+	return oTFR, nil
+}