diff --git a/go.mod b/go.mod index e3c7c50e72..bdcd1fe112 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/avct/uasurfer v0.0.0-20191028135549-26b5daa857f1 github.com/aws/aws-sdk-go v1.38.67 github.com/blang/semver v3.5.1+incompatible - github.com/blevesearch/bleve v1.0.14 + github.com/blevesearch/bleve/v2 v2.1.0 github.com/cespare/xxhash/v2 v2.1.1 github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/dgryski/dgoogauth v0.0.0-20190221195224-5a805980a5f3 @@ -114,7 +114,6 @@ require ( github.com/ulikunitz/xz v0.5.10 // indirect github.com/vmihailenco/msgpack/v5 v5.3.4 github.com/wiggin77/merror v1.0.3 - github.com/willf/bitset v1.1.11 // indirect github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c github.com/yuin/goldmark v1.3.8 go.etcd.io/bbolt v1.3.6 // indirect diff --git a/go.sum b/go.sum index aa01f4795a..de46471b2d 100644 --- a/go.sum +++ b/go.sum @@ -77,6 +77,7 @@ github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBK github.com/PuerkitoBio/goquery v1.7.0 h1:O5SP3b9JWqMSVMG69zMfj577zwkSNpxrFf7ybS74eiw= github.com/PuerkitoBio/goquery v1.7.0/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= +github.com/RoaringBitmap/roaring v0.7.3/go.mod h1:jdT9ykXwHFNdJbEtxePexlFYH9LXucApeS0/+/g+p1I= github.com/RoaringBitmap/roaring v0.8.0 h1:3naGkGvYAANr3ntyV+OpyhH+/rDaE2om9coNpIIdWAA= github.com/RoaringBitmap/roaring v0.8.0/go.mod h1:jdT9ykXwHFNdJbEtxePexlFYH9LXucApeS0/+/g+p1I= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= @@ -132,29 +133,35 @@ github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwj github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/blevesearch/bleve v1.0.14 h1:Q8r+fHTt35jtGXJUM0ULwM3Tzg+MRfyai4ZkWDy2xO4= -github.com/blevesearch/bleve v1.0.14/go.mod h1:e/LJTr+E7EaoVdkQZTfoz7dt4KoDNvDbLb8MSKuNTLQ= -github.com/blevesearch/blevex v1.0.0 h1:pnilj2Qi3YSEGdWgLj1Pn9Io7ukfXPoQcpAI1Bv8n/o= -github.com/blevesearch/blevex v1.0.0/go.mod h1:2rNVqoG2BZI8t1/P1awgTKnGlx5MP9ZbtEciQaNhswc= -github.com/blevesearch/cld2 v0.0.0-20200327141045-8b5f551d37f5/go.mod h1:PN0QNTLs9+j1bKy3d/GB/59wsNBFC4sWLWG3k69lWbc= +github.com/blevesearch/bleve/v2 v2.1.0 h1:SmD/DfWFrAP0TNrJagxsOR7nBt+6bUwos1MKzJInC4I= +github.com/blevesearch/bleve/v2 v2.1.0/go.mod h1:49zwlH3gtzAPNNzaaekkGz5T1lB/iDnzN4hS4W0yTHE= +github.com/blevesearch/bleve_index_api v1.0.0/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= +github.com/blevesearch/bleve_index_api v1.0.1 h1:nx9++0hnyiGOHJwQQYfsUGzpRdEVE5LsylmmngQvaFk= +github.com/blevesearch/bleve_index_api v1.0.1/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M= github.com/blevesearch/mmap-go v1.0.2 h1:JtMHb+FgQCTTYIhtMvimw15dJwu1Y5lrZDMOFXVWPk0= github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA= +github.com/blevesearch/scorch_segment_api/v2 v2.0.1 h1:fd+hPtZ8GsbqPK1HslGp7Vhoik4arZteA/IsCEgOisw= +github.com/blevesearch/scorch_segment_api/v2 v2.0.1/go.mod h1:lq7yK2jQy1yQjtjTfU931aVqz7pYxEudHaDwOt1tXfU= github.com/blevesearch/segment v0.9.0 h1:5lG7yBCx98or7gK2cHMKPukPZ/31Kag7nONpoBt22Ac= github.com/blevesearch/segment v0.9.0/go.mod h1:9PfHYUdQCgHktBgvtUOF4x+pc4/l8rdH0u5spnW85UQ= github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s= github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs= -github.com/blevesearch/zap/v11 v11.0.14 h1:IrDAvtlzDylh6H2QCmS0OGcN9Hpf6mISJlfKjcwJs7k= -github.com/blevesearch/zap/v11 v11.0.14/go.mod h1:MUEZh6VHGXv1PKx3WnCbdP404LGG2IZVa/L66pyFwnY= -github.com/blevesearch/zap/v12 v12.0.14 h1:2o9iRtl1xaRjsJ1xcqTyLX414qPAwykHNV7wNVmbp3w= -github.com/blevesearch/zap/v12 v12.0.14/go.mod h1:rOnuZOiMKPQj18AEKEHJxuI14236tTQ1ZJz4PAnWlUg= -github.com/blevesearch/zap/v13 v13.0.6 h1:r+VNSVImi9cBhTNNR+Kfl5uiGy8kIbb0JMz/h8r6+O4= -github.com/blevesearch/zap/v13 v13.0.6/go.mod h1:L89gsjdRKGyGrRN6nCpIScCvvkyxvmeDCwZRcjjPCrw= -github.com/blevesearch/zap/v14 v14.0.5 h1:NdcT+81Nvmp2zL+NhwSvGSLh7xNgGL8QRVZ67njR0NU= -github.com/blevesearch/zap/v14 v14.0.5/go.mod h1:bWe8S7tRrSBTIaZ6cLRbgNH4TUDaC9LZSpRGs85AsGY= -github.com/blevesearch/zap/v15 v15.0.3 h1:Ylj8Oe+mo0P25tr9iLPp33lN6d4qcztGjaIsP51UxaY= -github.com/blevesearch/zap/v15 v15.0.3/go.mod h1:iuwQrImsh1WjWJ0Ue2kBqY83a0rFtJTqfa9fp1rbVVU= +github.com/blevesearch/upsidedown_store_api v1.0.1 h1:1SYRwyoFLwG3sj0ed89RLtM15amfX2pXlYbFOnF8zNU= +github.com/blevesearch/upsidedown_store_api v1.0.1/go.mod h1:MQDVGpHZrpe3Uy26zJBf/a8h0FZY6xJbthIMm8myH2Q= +github.com/blevesearch/vellum v1.0.5 h1:L5dJ7hKauRVbuH7I8uqLeSK92CPPY6FfrbAmLhAug8A= +github.com/blevesearch/vellum v1.0.5/go.mod h1:atE0EH3fvk43zzS7t1YNdNC7DbmcC3uz+eMD5xZ2OyQ= +github.com/blevesearch/zapx/v11 v11.2.2 h1:yeHRnGA4UPxVm1roONbp7VRm+SUx95AleE4rU8w4pc4= +github.com/blevesearch/zapx/v11 v11.2.2/go.mod h1:qunXXAB8awrvPgnHdqbPFMW01N93bhKFxkEWxYQgp8w= +github.com/blevesearch/zapx/v12 v12.2.2 h1:aK6r0DbMMI8+MnrqkmFwWxUr8ZwVwQf8owbdVv7uhKs= +github.com/blevesearch/zapx/v12 v12.2.2/go.mod h1:6reJkgolYR1r7GC6SwbuRGhvMWin+Ou/n2Cd7DdvYzY= +github.com/blevesearch/zapx/v13 v13.2.2 h1:6oa7kZhywrRT3CeuN0bIZC6EviQkbPOQpvdR5pnxCks= +github.com/blevesearch/zapx/v13 v13.2.2/go.mod h1:EeLDRSUIMxBFwD4hZxVioHLR7gpY4VF7kdUoImp4Vy8= +github.com/blevesearch/zapx/v14 v14.2.2 h1:lRZCBvQIByW8F+mCoY4uA6Uen2DCj7K+wCh5nVKNxew= +github.com/blevesearch/zapx/v14 v14.2.2/go.mod h1:zb01unR63/DfV2nvyAvgj64K8AKtj7WP6w23jtuHntQ= +github.com/blevesearch/zapx/v15 v15.2.2 h1:5+oWWAQTV3M0UNor05qrZujzxIXVuwtYU3ppZ7Y1aNI= +github.com/blevesearch/zapx/v15 v15.2.2/go.mod h1:I4QVJ432LKkZyNK1kZkh3OweKa+NSblZzIF0YSSExak= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= @@ -193,17 +200,11 @@ github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k= github.com/couchbase/moss v0.1.0/go.mod h1:9MaHIaRuy9pvLPUJxB8sh8OrLfyDczECVL37grCIubs= -github.com/couchbase/vellum v1.0.2 h1:BrbP0NKiyDdndMPec8Jjhy0U47CZ0Lgx3xUC2r9rZqw= -github.com/couchbase/vellum v1.0.2/go.mod h1:FcwrEivFpNi24R3jLOs3n+fs5RnuQnQqCLBJ1uAg1W4= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d h1:SwD98825d6bdB+pEuTxWOXiSjBrHdOl/UVp75eI7JT8= -github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8= github.com/cznic/mathutil v0.0.0-20180504122225-ca4c9f2c1369/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= -github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= -github.com/cznic/strutil v0.0.0-20181122101858-275e90344537/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -252,9 +253,6 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= -github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= -github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= -github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.12.0 h1:mRhaKNwANqRgUBGKmnI5ZxEk7QXmjQeCcuYFMX2bfcc= @@ -488,7 +486,6 @@ github.com/hashicorp/yamux v0.0.0-20210316155119-a95892c5f864 h1:Y4V+SFe7d3iH+9p github.com/hashicorp/yamux v0.0.0-20210316155119-a95892c5f864/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ikawaha/kagome.ipadic v1.1.2/go.mod h1:DPSBbU0czaJhAb/5uKQZHMc9MTVRpDugJfX+HddPHHg= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= @@ -533,8 +530,6 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U= -github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jmoiron/sqlx v1.3.4 h1:wv+0IJZfL5z0uZoUjlpKgHkgaFSYD+r9CfrXjEXsO7w= github.com/jmoiron/sqlx v1.3.4/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= @@ -844,7 +839,6 @@ github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqn github.com/reflog/dateconstraints v0.2.1 h1:Hz1n2Q1vEm0Rj5gciDQcCN1iPBwfFjxUJy32NknGP/s= github.com/reflog/dateconstraints v0.2.1/go.mod h1:Ax8AxTBcJc3E/oVS2hd2j7RDM/5MDtuPwuR7lIHtPLo= github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -960,12 +954,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= -github.com/tebeka/snowball v0.4.2/go.mod h1:4IfL14h1lvwZcp1sfXuuc7/7yCsvVffTWxWxCLfFpYg= -github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok= -github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8= github.com/throttled/throttled v2.2.5+incompatible h1:65UB52X0qNTYiT0Sohp8qLYVFwZQPDw85uSa65OljjQ= github.com/throttled/throttled v2.2.5+incompatible/go.mod h1:0BjlrEGQmvxps+HuXLsyRdqpSRvJpq0PNIsOtqP9Nos= github.com/tidwall/gjson v1.8.0 h1:Qt+orfosKn0rbNTZqHYDqBrmm3UDA4KRkv70fDzG+PQ= @@ -1012,8 +1002,6 @@ github.com/wiggin77/merror v1.0.3/go.mod h1:H2ETSu7/bPE0Ymf4bEwdUoo73OOEkdClnoRi github.com/wiggin77/srslog v1.0.1 h1:gA2XjSMy3DrRdX9UqLuDtuVAAshb8bE1NhX1YK0Qe+8= github.com/wiggin77/srslog v1.0.1/go.mod h1:fehkyYDq1QfuYn60TDPu9YdY2bB85VUW2mvN1WynEls= github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= -github.com/willf/bitset v1.1.11 h1:N7Z7E9UvjW+sGsEl7k/SJrvY2reP1A07MrGuCjIOjRE= -github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= github.com/xanzy/go-gitlab v0.15.0/go.mod h1:8zdQa/ri1dfn8eS3Ir1SyfvOKlw7WBJ8DVThkpGiXrs= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= diff --git a/services/searchengine/bleveengine/bleve.go b/services/searchengine/bleveengine/bleve.go index 3c4fb139a5..13fa708c92 100644 --- a/services/searchengine/bleveengine/bleve.go +++ b/services/searchengine/bleveengine/bleve.go @@ -12,10 +12,10 @@ import ( "sync/atomic" "time" - "github.com/blevesearch/bleve" - "github.com/blevesearch/bleve/analysis/analyzer/keyword" - "github.com/blevesearch/bleve/analysis/analyzer/standard" - "github.com/blevesearch/bleve/mapping" + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" + "github.com/blevesearch/bleve/v2/analysis/analyzer/standard" + "github.com/blevesearch/bleve/v2/mapping" "github.com/mattermost/mattermost-server/v6/jobs" "github.com/mattermost/mattermost-server/v6/model" @@ -134,7 +134,10 @@ func (b *BleveEngine) createOrOpenIndex(indexName string, mapping *mapping.Index return index, nil } - index, err := bleve.New(indexPath, mapping) + index, err := bleve.NewUsing(indexPath, mapping, "scorch", "scorch", map[string]interface{}{ + "forceSegmentType": "zap", + "forceSegmentVersion": 15, + }) if err != nil { return nil, err } diff --git a/services/searchengine/bleveengine/bleve_test.go b/services/searchengine/bleveengine/bleve_test.go index ba77f67257..6bdba71f85 100644 --- a/services/searchengine/bleveengine/bleve_test.go +++ b/services/searchengine/bleveengine/bleve_test.go @@ -8,7 +8,7 @@ import ( "os" "testing" - "github.com/blevesearch/bleve" + "github.com/blevesearch/bleve/v2" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -124,7 +124,7 @@ func (s *BleveEngineTestSuite) TestDeleteChannelPosts() { doc, err := s.BleveEngine.PostIndex.Document(postToAvoid.Id) require.NoError(s.T(), err) - require.Equal(s.T(), postToAvoid.Id, doc.ID) + require.Equal(s.T(), postToAvoid.Id, doc.ID()) numberDocs, err := s.BleveEngine.PostIndex.DocCount() require.NoError(s.T(), err) require.Equal(s.T(), 1, int(numberDocs)) @@ -170,7 +170,7 @@ func (s *BleveEngineTestSuite) TestDeleteUserPosts() { doc, err := s.BleveEngine.PostIndex.Document(postToAvoid.Id) require.NoError(s.T(), err) - require.Equal(s.T(), postToAvoid.Id, doc.ID) + require.Equal(s.T(), postToAvoid.Id, doc.ID()) numberDocs, err := s.BleveEngine.PostIndex.DocCount() require.NoError(s.T(), err) require.Equal(s.T(), 1, int(numberDocs)) @@ -220,7 +220,7 @@ func (s *BleveEngineTestSuite) TestDeletePosts() { doc, err := s.BleveEngine.PostIndex.Document(postToAvoid.Id) require.NoError(s.T(), err) - require.Equal(s.T(), postToAvoid.Id, doc.ID) + require.Equal(s.T(), postToAvoid.Id, doc.ID()) numberDocs, err := s.BleveEngine.PostIndex.DocCount() require.NoError(s.T(), err) require.Equal(s.T(), 1, int(numberDocs)) diff --git a/services/searchengine/bleveengine/search.go b/services/searchengine/bleveengine/search.go index 85cd4eaae4..87ac5ab199 100644 --- a/services/searchengine/bleveengine/search.go +++ b/services/searchengine/bleveengine/search.go @@ -7,8 +7,8 @@ import ( "net/http" "strings" - "github.com/blevesearch/bleve" - "github.com/blevesearch/bleve/search/query" + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/search/query" "github.com/mattermost/mattermost-server/v6/model" "github.com/mattermost/mattermost-server/v6/shared/mlog" diff --git a/vendor/github.com/blevesearch/bleve/go.mod b/vendor/github.com/blevesearch/bleve/go.mod deleted file mode 100644 index 6300aea17f..0000000000 --- a/vendor/github.com/blevesearch/bleve/go.mod +++ /dev/null @@ -1,27 +0,0 @@ -module github.com/blevesearch/bleve - -go 1.13 - -require ( - github.com/RoaringBitmap/roaring v0.4.23 - github.com/blevesearch/blevex v1.0.0 - github.com/blevesearch/go-porterstemmer v1.0.3 - github.com/blevesearch/segment v0.9.0 - github.com/blevesearch/snowballstem v0.9.0 - github.com/blevesearch/zap/v11 v11.0.14 - github.com/blevesearch/zap/v12 v12.0.14 - github.com/blevesearch/zap/v13 v13.0.6 - github.com/blevesearch/zap/v14 v14.0.5 - github.com/blevesearch/zap/v15 v15.0.3 - github.com/couchbase/moss v0.1.0 - github.com/couchbase/vellum v1.0.2 - github.com/golang/protobuf v1.3.2 - github.com/kljensen/snowball v0.6.0 - github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 - github.com/spf13/cobra v0.0.5 - github.com/steveyen/gtreap v0.1.0 - github.com/syndtr/goleveldb v1.0.0 - github.com/willf/bitset v1.1.10 - go.etcd.io/bbolt v1.3.5 - golang.org/x/text v0.3.0 -) diff --git a/vendor/github.com/blevesearch/bleve/index/analysis.go b/vendor/github.com/blevesearch/bleve/index/analysis.go deleted file mode 100644 index 82883af019..0000000000 --- a/vendor/github.com/blevesearch/bleve/index/analysis.go +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright (c) 2015 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package index - -import ( - "reflect" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/size" -) - -var reflectStaticSizeAnalysisResult int - -func init() { - var ar AnalysisResult - reflectStaticSizeAnalysisResult = int(reflect.TypeOf(ar).Size()) -} - -type IndexRow interface { - KeySize() int - KeyTo([]byte) (int, error) - Key() []byte - - ValueSize() int - ValueTo([]byte) (int, error) - Value() []byte -} - -type AnalysisResult struct { - DocID string - Rows []IndexRow - - // scorch - Document *document.Document - Analyzed []analysis.TokenFrequencies - Length []int -} - -func (a *AnalysisResult) Size() int { - rv := reflectStaticSizeAnalysisResult - for _, analyzedI := range a.Analyzed { - rv += analyzedI.Size() - } - rv += len(a.Length) * size.SizeOfInt - return rv -} - -type AnalysisWork struct { - i Index - d *document.Document - rc chan *AnalysisResult -} - -func NewAnalysisWork(i Index, d *document.Document, rc chan *AnalysisResult) *AnalysisWork { - return &AnalysisWork{ - i: i, - d: d, - rc: rc, - } -} - -type AnalysisQueue struct { - queue chan *AnalysisWork - done chan struct{} -} - -func (q *AnalysisQueue) Queue(work *AnalysisWork) { - q.queue <- work -} - -func (q *AnalysisQueue) Close() { - close(q.done) -} - -func NewAnalysisQueue(numWorkers int) *AnalysisQueue { - rv := AnalysisQueue{ - queue: make(chan *AnalysisWork), - done: make(chan struct{}), - } - for i := 0; i < numWorkers; i++ { - go AnalysisWorker(rv) - } - return &rv -} - -func AnalysisWorker(q AnalysisQueue) { - // read work off the queue - for { - select { - case <-q.done: - return - case w := <-q.queue: - r := w.i.Analyze(w.d) - w.rc <- r - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/segment/empty.go b/vendor/github.com/blevesearch/bleve/index/scorch/segment/empty.go deleted file mode 100644 index 340db73a67..0000000000 --- a/vendor/github.com/blevesearch/bleve/index/scorch/segment/empty.go +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package segment - -import ( - "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index" - "github.com/couchbase/vellum" -) - -type EmptySegment struct{} - -func (e *EmptySegment) Dictionary(field string) (TermDictionary, error) { - return &EmptyDictionary{}, nil -} - -func (e *EmptySegment) VisitDocument(num uint64, visitor DocumentFieldValueVisitor) error { - return nil -} - -func (e *EmptySegment) DocID(num uint64) ([]byte, error) { - return nil, nil -} - -func (e *EmptySegment) Count() uint64 { - return 0 -} - -func (e *EmptySegment) DocNumbers([]string) (*roaring.Bitmap, error) { - r := roaring.NewBitmap() - return r, nil -} - -func (e *EmptySegment) Fields() []string { - return []string{} -} - -func (e *EmptySegment) Close() error { - return nil -} - -func (e *EmptySegment) Size() uint64 { - return 0 -} - -func (e *EmptySegment) AddRef() { -} - -func (e *EmptySegment) DecRef() error { - return nil -} - -type EmptyDictionary struct{} - -func (e *EmptyDictionary) PostingsList(term []byte, - except *roaring.Bitmap, prealloc PostingsList) (PostingsList, error) { - return &EmptyPostingsList{}, nil -} - -func (e *EmptyDictionary) Iterator() DictionaryIterator { - return &EmptyDictionaryIterator{} -} - -func (e *EmptyDictionary) PrefixIterator(prefix string) DictionaryIterator { - return &EmptyDictionaryIterator{} -} - -func (e *EmptyDictionary) RangeIterator(start, end string) DictionaryIterator { - return &EmptyDictionaryIterator{} -} - -func (e *EmptyDictionary) AutomatonIterator(a vellum.Automaton, - startKeyInclusive, endKeyExclusive []byte) DictionaryIterator { - return &EmptyDictionaryIterator{} -} - -func (e *EmptyDictionary) OnlyIterator(onlyTerms [][]byte, - includeCount bool) DictionaryIterator { - return &EmptyDictionaryIterator{} -} - -func (e *EmptyDictionary) Contains(key []byte) (bool, error) { - return false, nil -} - -type EmptyDictionaryIterator struct{} - -func (e *EmptyDictionaryIterator) Next() (*index.DictEntry, error) { - return nil, nil -} - -func (e *EmptyDictionaryIterator) Contains(key []byte) (bool, error) { - return false, nil -} - -type EmptyPostingsList struct{} - -func (e *EmptyPostingsList) Iterator(includeFreq, includeNorm, includeLocations bool, - prealloc PostingsIterator) PostingsIterator { - return &EmptyPostingsIterator{} -} - -func (e *EmptyPostingsList) Size() int { - return 0 -} - -func (e *EmptyPostingsList) Count() uint64 { - return 0 -} - -type EmptyPostingsIterator struct{} - -func (e *EmptyPostingsIterator) Next() (Posting, error) { - return nil, nil -} - -func (e *EmptyPostingsIterator) Advance(uint64) (Posting, error) { - return nil, nil -} - -func (e *EmptyPostingsIterator) Size() int { - return 0 -} - -var AnEmptyPostingsIterator = &EmptyPostingsIterator{} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/segment/int.go b/vendor/github.com/blevesearch/bleve/index/scorch/segment/int.go deleted file mode 100644 index 55299d8f7a..0000000000 --- a/vendor/github.com/blevesearch/bleve/index/scorch/segment/int.go +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright 2014 The Cockroach Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. See the License for the specific language governing -// permissions and limitations under the License. - -// This code originated from: -// https://github.com/cockroachdb/cockroach/blob/2dd65dde5d90c157f4b93f92502ca1063b904e1d/pkg/util/encoding/encoding.go - -// Modified to not use pkg/errors - -package segment - -import ( - "errors" - "fmt" -) - -const ( - MaxVarintSize = 9 - - // IntMin is chosen such that the range of int tags does not overlap the - // ascii character set that is frequently used in testing. - IntMin = 0x80 // 128 - intMaxWidth = 8 - intZero = IntMin + intMaxWidth // 136 - intSmall = IntMax - intZero - intMaxWidth // 109 - // IntMax is the maximum int tag value. - IntMax = 0xfd // 253 -) - -// EncodeUvarintAscending encodes the uint64 value using a variable length -// (length-prefixed) representation. The length is encoded as a single -// byte indicating the number of encoded bytes (-8) to follow. See -// EncodeVarintAscending for rationale. The encoded bytes are appended to the -// supplied buffer and the final buffer is returned. -func EncodeUvarintAscending(b []byte, v uint64) []byte { - switch { - case v <= intSmall: - return append(b, intZero+byte(v)) - case v <= 0xff: - return append(b, IntMax-7, byte(v)) - case v <= 0xffff: - return append(b, IntMax-6, byte(v>>8), byte(v)) - case v <= 0xffffff: - return append(b, IntMax-5, byte(v>>16), byte(v>>8), byte(v)) - case v <= 0xffffffff: - return append(b, IntMax-4, byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) - case v <= 0xffffffffff: - return append(b, IntMax-3, byte(v>>32), byte(v>>24), byte(v>>16), byte(v>>8), - byte(v)) - case v <= 0xffffffffffff: - return append(b, IntMax-2, byte(v>>40), byte(v>>32), byte(v>>24), byte(v>>16), - byte(v>>8), byte(v)) - case v <= 0xffffffffffffff: - return append(b, IntMax-1, byte(v>>48), byte(v>>40), byte(v>>32), byte(v>>24), - byte(v>>16), byte(v>>8), byte(v)) - default: - return append(b, IntMax, byte(v>>56), byte(v>>48), byte(v>>40), byte(v>>32), - byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) - } -} - -// DecodeUvarintAscending decodes a varint encoded uint64 from the input -// buffer. The remainder of the input buffer and the decoded uint64 -// are returned. -func DecodeUvarintAscending(b []byte) ([]byte, uint64, error) { - if len(b) == 0 { - return nil, 0, fmt.Errorf("insufficient bytes to decode uvarint value") - } - length := int(b[0]) - intZero - b = b[1:] // skip length byte - if length <= intSmall { - return b, uint64(length), nil - } - length -= intSmall - if length < 0 || length > 8 { - return nil, 0, fmt.Errorf("invalid uvarint length of %d", length) - } else if len(b) < length { - return nil, 0, fmt.Errorf("insufficient bytes to decode uvarint value: %q", b) - } - var v uint64 - // It is faster to range over the elements in a slice than to index - // into the slice on each loop iteration. - for _, t := range b[:length] { - v = (v << 8) | uint64(t) - } - return b[length:], v, nil -} - -// ------------------------------------------------------------ - -type MemUvarintReader struct { - C int // index of next byte to read from S - S []byte -} - -func NewMemUvarintReader(s []byte) *MemUvarintReader { - return &MemUvarintReader{S: s} -} - -// Len returns the number of unread bytes. -func (r *MemUvarintReader) Len() int { - n := len(r.S) - r.C - if n < 0 { - return 0 - } - return n -} - -var ErrMemUvarintReaderOverflow = errors.New("MemUvarintReader overflow") - -// ReadUvarint reads an encoded uint64. The original code this was -// based on is at encoding/binary/ReadUvarint(). -func (r *MemUvarintReader) ReadUvarint() (uint64, error) { - var x uint64 - var s uint - var C = r.C - var S = r.S - - for { - b := S[C] - C++ - - if b < 0x80 { - r.C = C - - // why 63? The original code had an 'i += 1' loop var and - // checked for i > 9 || i == 9 ...; but, we no longer - // check for the i var, but instead check here for s, - // which is incremented by 7. So, 7*9 == 63. - // - // why the "extra" >= check? The normal case is that s < - // 63, so we check this single >= guard first so that we - // hit the normal, nil-error return pathway sooner. - if s >= 63 && (s > 63 || s == 63 && b > 1) { - return 0, ErrMemUvarintReaderOverflow - } - - return x | uint64(b)< 0 +} diff --git a/vendor/github.com/blevesearch/bleve/document/field.go b/vendor/github.com/blevesearch/bleve/v2/document/field.go similarity index 87% rename from vendor/github.com/blevesearch/bleve/document/field.go rename to vendor/github.com/blevesearch/bleve/v2/document/field.go index 2fe9166985..eb104e2dfe 100644 --- a/vendor/github.com/blevesearch/bleve/document/field.go +++ b/vendor/github.com/blevesearch/bleve/v2/document/field.go @@ -15,7 +15,7 @@ package document import ( - "github.com/blevesearch/bleve/analysis" + index "github.com/blevesearch/bleve_index_api" ) type Field interface { @@ -28,8 +28,8 @@ type Field interface { // arrays, ArrayPositions returns 2 indices used to resolve "doc2" value in // "doc1", then "field" in "doc2". ArrayPositions() []uint64 - Options() IndexingOptions - Analyze() (int, analysis.TokenFrequencies) + Options() index.FieldIndexingOptions + Analyze() Value() []byte // NumPlainTextBytes should return the number of plain text bytes @@ -38,4 +38,8 @@ type Field interface { NumPlainTextBytes() uint64 Size() int + + EncodedFieldType() byte + AnalyzedLength() int + AnalyzedTokenFrequencies() index.TokenFrequencies } diff --git a/vendor/github.com/blevesearch/bleve/document/field_boolean.go b/vendor/github.com/blevesearch/bleve/v2/document/field_boolean.go similarity index 76% rename from vendor/github.com/blevesearch/bleve/document/field_boolean.go rename to vendor/github.com/blevesearch/bleve/v2/document/field_boolean.go index 6864b16f44..fdf3cc0e53 100644 --- a/vendor/github.com/blevesearch/bleve/document/field_boolean.go +++ b/vendor/github.com/blevesearch/bleve/v2/document/field_boolean.go @@ -18,8 +18,9 @@ import ( "fmt" "reflect" - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeBooleanField int @@ -29,14 +30,16 @@ func init() { reflectStaticSizeBooleanField = int(reflect.TypeOf(f).Size()) } -const DefaultBooleanIndexingOptions = StoreField | IndexField | DocValues +const DefaultBooleanIndexingOptions = index.StoreField | index.IndexField | index.DocValues type BooleanField struct { name string arrayPositions []uint64 - options IndexingOptions + options index.FieldIndexingOptions value []byte numPlainTextBytes uint64 + length int + frequencies index.TokenFrequencies } func (b *BooleanField) Size() int { @@ -54,11 +57,11 @@ func (b *BooleanField) ArrayPositions() []uint64 { return b.arrayPositions } -func (b *BooleanField) Options() IndexingOptions { +func (b *BooleanField) Options() index.FieldIndexingOptions { return b.options } -func (b *BooleanField) Analyze() (int, analysis.TokenFrequencies) { +func (b *BooleanField) Analyze() { tokens := make(analysis.TokenStream, 0) tokens = append(tokens, &analysis.Token{ Start: 0, @@ -68,9 +71,8 @@ func (b *BooleanField) Analyze() (int, analysis.TokenFrequencies) { Type: analysis.Boolean, }) - fieldLength := len(tokens) - tokenFreqs := analysis.TokenFrequency(tokens, b.arrayPositions, b.options.IncludeTermVectors()) - return fieldLength, tokenFreqs + b.length = len(tokens) + b.frequencies = analysis.TokenFrequency(tokens, b.arrayPositions, b.options) } func (b *BooleanField) Value() []byte { @@ -92,6 +94,18 @@ func (b *BooleanField) NumPlainTextBytes() uint64 { return b.numPlainTextBytes } +func (b *BooleanField) EncodedFieldType() byte { + return 'b' +} + +func (b *BooleanField) AnalyzedLength() int { + return b.length +} + +func (b *BooleanField) AnalyzedTokenFrequencies() index.TokenFrequencies { + return b.frequencies +} + func NewBooleanFieldFromBytes(name string, arrayPositions []uint64, value []byte) *BooleanField { return &BooleanField{ name: name, @@ -106,7 +120,7 @@ func NewBooleanField(name string, arrayPositions []uint64, b bool) *BooleanField return NewBooleanFieldWithIndexingOptions(name, arrayPositions, b, DefaultNumericIndexingOptions) } -func NewBooleanFieldWithIndexingOptions(name string, arrayPositions []uint64, b bool, options IndexingOptions) *BooleanField { +func NewBooleanFieldWithIndexingOptions(name string, arrayPositions []uint64, b bool, options index.FieldIndexingOptions) *BooleanField { numPlainTextBytes := 5 v := []byte("F") if b { diff --git a/vendor/github.com/blevesearch/bleve/document/field_composite.go b/vendor/github.com/blevesearch/bleve/v2/document/field_composite.go similarity index 78% rename from vendor/github.com/blevesearch/bleve/document/field_composite.go rename to vendor/github.com/blevesearch/bleve/v2/document/field_composite.go index a8285880fd..8c47643f5b 100644 --- a/vendor/github.com/blevesearch/bleve/document/field_composite.go +++ b/vendor/github.com/blevesearch/bleve/v2/document/field_composite.go @@ -17,8 +17,8 @@ package document import ( "reflect" - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeCompositeField int @@ -28,30 +28,30 @@ func init() { reflectStaticSizeCompositeField = int(reflect.TypeOf(cf).Size()) } -const DefaultCompositeIndexingOptions = IndexField +const DefaultCompositeIndexingOptions = index.IndexField type CompositeField struct { name string includedFields map[string]bool excludedFields map[string]bool defaultInclude bool - options IndexingOptions + options index.FieldIndexingOptions totalLength int - compositeFrequencies analysis.TokenFrequencies + compositeFrequencies index.TokenFrequencies } func NewCompositeField(name string, defaultInclude bool, include []string, exclude []string) *CompositeField { return NewCompositeFieldWithIndexingOptions(name, defaultInclude, include, exclude, DefaultCompositeIndexingOptions) } -func NewCompositeFieldWithIndexingOptions(name string, defaultInclude bool, include []string, exclude []string, options IndexingOptions) *CompositeField { +func NewCompositeFieldWithIndexingOptions(name string, defaultInclude bool, include []string, exclude []string, options index.FieldIndexingOptions) *CompositeField { rv := &CompositeField{ name: name, options: options, defaultInclude: defaultInclude, includedFields: make(map[string]bool, len(include)), excludedFields: make(map[string]bool, len(exclude)), - compositeFrequencies: make(analysis.TokenFrequencies), + compositeFrequencies: make(index.TokenFrequencies), } for _, i := range include { @@ -87,12 +87,11 @@ func (c *CompositeField) ArrayPositions() []uint64 { return []uint64{} } -func (c *CompositeField) Options() IndexingOptions { +func (c *CompositeField) Options() index.FieldIndexingOptions { return c.options } -func (c *CompositeField) Analyze() (int, analysis.TokenFrequencies) { - return c.totalLength, c.compositeFrequencies +func (c *CompositeField) Analyze() { } func (c *CompositeField) Value() []byte { @@ -116,9 +115,21 @@ func (c *CompositeField) includesField(field string) bool { return shouldInclude } -func (c *CompositeField) Compose(field string, length int, freq analysis.TokenFrequencies) { +func (c *CompositeField) Compose(field string, length int, freq index.TokenFrequencies) { if c.includesField(field) { c.totalLength += length c.compositeFrequencies.MergeAll(field, freq) } } + +func (c *CompositeField) EncodedFieldType() byte { + return 'c' +} + +func (c *CompositeField) AnalyzedLength() int { + return c.totalLength +} + +func (c *CompositeField) AnalyzedTokenFrequencies() index.TokenFrequencies { + return c.compositeFrequencies +} diff --git a/vendor/github.com/blevesearch/bleve/document/field_datetime.go b/vendor/github.com/blevesearch/bleve/v2/document/field_datetime.go similarity index 80% rename from vendor/github.com/blevesearch/bleve/document/field_datetime.go rename to vendor/github.com/blevesearch/bleve/v2/document/field_datetime.go index 583b44cdeb..6506405501 100644 --- a/vendor/github.com/blevesearch/bleve/document/field_datetime.go +++ b/vendor/github.com/blevesearch/bleve/v2/document/field_datetime.go @@ -20,9 +20,10 @@ import ( "reflect" "time" - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/numeric" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeDateTimeField int @@ -32,7 +33,7 @@ func init() { reflectStaticSizeDateTimeField = int(reflect.TypeOf(f).Size()) } -const DefaultDateTimeIndexingOptions = StoreField | IndexField | DocValues +const DefaultDateTimeIndexingOptions = index.StoreField | index.IndexField | index.DocValues const DefaultDateTimePrecisionStep uint = 4 var MinTimeRepresentable = time.Unix(0, math.MinInt64) @@ -41,9 +42,11 @@ var MaxTimeRepresentable = time.Unix(0, math.MaxInt64) type DateTimeField struct { name string arrayPositions []uint64 - options IndexingOptions + options index.FieldIndexingOptions value numeric.PrefixCoded numPlainTextBytes uint64 + length int + frequencies index.TokenFrequencies } func (n *DateTimeField) Size() int { @@ -60,11 +63,23 @@ func (n *DateTimeField) ArrayPositions() []uint64 { return n.arrayPositions } -func (n *DateTimeField) Options() IndexingOptions { +func (n *DateTimeField) Options() index.FieldIndexingOptions { return n.options } -func (n *DateTimeField) Analyze() (int, analysis.TokenFrequencies) { +func (n *DateTimeField) EncodedFieldType() byte { + return 'd' +} + +func (n *DateTimeField) AnalyzedLength() int { + return n.length +} + +func (n *DateTimeField) AnalyzedTokenFrequencies() index.TokenFrequencies { + return n.frequencies +} + +func (n *DateTimeField) Analyze() { tokens := make(analysis.TokenStream, 0) tokens = append(tokens, &analysis.Token{ Start: 0, @@ -95,9 +110,8 @@ func (n *DateTimeField) Analyze() (int, analysis.TokenFrequencies) { } } - fieldLength := len(tokens) - tokenFreqs := analysis.TokenFrequency(tokens, n.arrayPositions, n.options.IncludeTermVectors()) - return fieldLength, tokenFreqs + n.length = len(tokens) + n.frequencies = analysis.TokenFrequency(tokens, n.arrayPositions, n.options) } func (n *DateTimeField) Value() []byte { @@ -134,7 +148,7 @@ func NewDateTimeField(name string, arrayPositions []uint64, dt time.Time) (*Date return NewDateTimeFieldWithIndexingOptions(name, arrayPositions, dt, DefaultDateTimeIndexingOptions) } -func NewDateTimeFieldWithIndexingOptions(name string, arrayPositions []uint64, dt time.Time, options IndexingOptions) (*DateTimeField, error) { +func NewDateTimeFieldWithIndexingOptions(name string, arrayPositions []uint64, dt time.Time, options index.FieldIndexingOptions) (*DateTimeField, error) { if canRepresent(dt) { dtInt64 := dt.UnixNano() prefixCoded := numeric.MustNewPrefixCodedInt64(dtInt64, 0) diff --git a/vendor/github.com/blevesearch/bleve/document/field_geopoint.go b/vendor/github.com/blevesearch/bleve/v2/document/field_geopoint.go similarity index 80% rename from vendor/github.com/blevesearch/bleve/document/field_geopoint.go rename to vendor/github.com/blevesearch/bleve/v2/document/field_geopoint.go index 91fe23f96e..89de4e3bfa 100644 --- a/vendor/github.com/blevesearch/bleve/document/field_geopoint.go +++ b/vendor/github.com/blevesearch/bleve/v2/document/field_geopoint.go @@ -18,10 +18,11 @@ import ( "fmt" "reflect" - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/geo" - "github.com/blevesearch/bleve/numeric" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeGeoPointField int @@ -36,9 +37,11 @@ var GeoPrecisionStep uint = 9 type GeoPointField struct { name string arrayPositions []uint64 - options IndexingOptions + options index.FieldIndexingOptions value numeric.PrefixCoded numPlainTextBytes uint64 + length int + frequencies index.TokenFrequencies } func (n *GeoPointField) Size() int { @@ -55,11 +58,23 @@ func (n *GeoPointField) ArrayPositions() []uint64 { return n.arrayPositions } -func (n *GeoPointField) Options() IndexingOptions { +func (n *GeoPointField) Options() index.FieldIndexingOptions { return n.options } -func (n *GeoPointField) Analyze() (int, analysis.TokenFrequencies) { +func (n *GeoPointField) EncodedFieldType() byte { + return 'g' +} + +func (n *GeoPointField) AnalyzedLength() int { + return n.length +} + +func (n *GeoPointField) AnalyzedTokenFrequencies() index.TokenFrequencies { + return n.frequencies +} + +func (n *GeoPointField) Analyze() { tokens := make(analysis.TokenStream, 0) tokens = append(tokens, &analysis.Token{ Start: 0, @@ -90,9 +105,8 @@ func (n *GeoPointField) Analyze() (int, analysis.TokenFrequencies) { } } - fieldLength := len(tokens) - tokenFreqs := analysis.TokenFrequency(tokens, n.arrayPositions, n.options.IncludeTermVectors()) - return fieldLength, tokenFreqs + n.length = len(tokens) + n.frequencies = analysis.TokenFrequency(tokens, n.arrayPositions, n.options) } func (n *GeoPointField) Value() []byte { @@ -137,7 +151,7 @@ func NewGeoPointField(name string, arrayPositions []uint64, lon, lat float64) *G return NewGeoPointFieldWithIndexingOptions(name, arrayPositions, lon, lat, DefaultNumericIndexingOptions) } -func NewGeoPointFieldWithIndexingOptions(name string, arrayPositions []uint64, lon, lat float64, options IndexingOptions) *GeoPointField { +func NewGeoPointFieldWithIndexingOptions(name string, arrayPositions []uint64, lon, lat float64, options index.FieldIndexingOptions) *GeoPointField { mhash := geo.MortonHash(lon, lat) prefixCoded := numeric.MustNewPrefixCodedInt64(int64(mhash), 0) return &GeoPointField{ diff --git a/vendor/github.com/blevesearch/bleve/document/field_numeric.go b/vendor/github.com/blevesearch/bleve/v2/document/field_numeric.go similarity index 79% rename from vendor/github.com/blevesearch/bleve/document/field_numeric.go rename to vendor/github.com/blevesearch/bleve/v2/document/field_numeric.go index 46c685e84e..a54b082b48 100644 --- a/vendor/github.com/blevesearch/bleve/document/field_numeric.go +++ b/vendor/github.com/blevesearch/bleve/v2/document/field_numeric.go @@ -18,9 +18,10 @@ import ( "fmt" "reflect" - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/numeric" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeNumericField int @@ -30,16 +31,18 @@ func init() { reflectStaticSizeNumericField = int(reflect.TypeOf(f).Size()) } -const DefaultNumericIndexingOptions = StoreField | IndexField | DocValues +const DefaultNumericIndexingOptions = index.StoreField | index.IndexField | index.DocValues const DefaultPrecisionStep uint = 4 type NumericField struct { name string arrayPositions []uint64 - options IndexingOptions + options index.FieldIndexingOptions value numeric.PrefixCoded numPlainTextBytes uint64 + length int + frequencies index.TokenFrequencies } func (n *NumericField) Size() int { @@ -56,11 +59,23 @@ func (n *NumericField) ArrayPositions() []uint64 { return n.arrayPositions } -func (n *NumericField) Options() IndexingOptions { +func (n *NumericField) Options() index.FieldIndexingOptions { return n.options } -func (n *NumericField) Analyze() (int, analysis.TokenFrequencies) { +func (n *NumericField) EncodedFieldType() byte { + return 'n' +} + +func (n *NumericField) AnalyzedLength() int { + return n.length +} + +func (n *NumericField) AnalyzedTokenFrequencies() index.TokenFrequencies { + return n.frequencies +} + +func (n *NumericField) Analyze() { tokens := make(analysis.TokenStream, 0) tokens = append(tokens, &analysis.Token{ Start: 0, @@ -91,9 +106,8 @@ func (n *NumericField) Analyze() (int, analysis.TokenFrequencies) { } } - fieldLength := len(tokens) - tokenFreqs := analysis.TokenFrequency(tokens, n.arrayPositions, n.options.IncludeTermVectors()) - return fieldLength, tokenFreqs + n.length = len(tokens) + n.frequencies = analysis.TokenFrequency(tokens, n.arrayPositions, n.options) } func (n *NumericField) Value() []byte { @@ -130,7 +144,7 @@ func NewNumericField(name string, arrayPositions []uint64, number float64) *Nume return NewNumericFieldWithIndexingOptions(name, arrayPositions, number, DefaultNumericIndexingOptions) } -func NewNumericFieldWithIndexingOptions(name string, arrayPositions []uint64, number float64, options IndexingOptions) *NumericField { +func NewNumericFieldWithIndexingOptions(name string, arrayPositions []uint64, number float64, options index.FieldIndexingOptions) *NumericField { numberInt64 := numeric.Float64ToInt64(number) prefixCoded := numeric.MustNewPrefixCodedInt64(numberInt64, 0) return &NumericField{ diff --git a/vendor/github.com/blevesearch/bleve/document/field_text.go b/vendor/github.com/blevesearch/bleve/v2/document/field_text.go similarity index 76% rename from vendor/github.com/blevesearch/bleve/document/field_text.go rename to vendor/github.com/blevesearch/bleve/v2/document/field_text.go index 6bd74c7127..924de532c0 100644 --- a/vendor/github.com/blevesearch/bleve/document/field_text.go +++ b/vendor/github.com/blevesearch/bleve/v2/document/field_text.go @@ -18,8 +18,9 @@ import ( "fmt" "reflect" - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeTextField int @@ -29,15 +30,17 @@ func init() { reflectStaticSizeTextField = int(reflect.TypeOf(f).Size()) } -const DefaultTextIndexingOptions = IndexField | DocValues +const DefaultTextIndexingOptions = index.IndexField | index.DocValues type TextField struct { name string arrayPositions []uint64 - options IndexingOptions + options index.FieldIndexingOptions analyzer *analysis.Analyzer value []byte numPlainTextBytes uint64 + length int + frequencies index.TokenFrequencies } func (t *TextField) Size() int { @@ -55,11 +58,23 @@ func (t *TextField) ArrayPositions() []uint64 { return t.arrayPositions } -func (t *TextField) Options() IndexingOptions { +func (t *TextField) Options() index.FieldIndexingOptions { return t.options } -func (t *TextField) Analyze() (int, analysis.TokenFrequencies) { +func (t *TextField) EncodedFieldType() byte { + return 't' +} + +func (t *TextField) AnalyzedLength() int { + return t.length +} + +func (t *TextField) AnalyzedTokenFrequencies() index.TokenFrequencies { + return t.frequencies +} + +func (t *TextField) Analyze() { var tokens analysis.TokenStream if t.analyzer != nil { bytesToAnalyze := t.Value() @@ -81,9 +96,8 @@ func (t *TextField) Analyze() (int, analysis.TokenFrequencies) { }, } } - fieldLength := len(tokens) // number of tokens in this doc field - tokenFreqs := analysis.TokenFrequency(tokens, t.arrayPositions, t.options.IncludeTermVectors()) - return fieldLength, tokenFreqs + t.length = len(tokens) // number of tokens in this doc field + t.frequencies = analysis.TokenFrequency(tokens, t.arrayPositions, t.options) } func (t *TextField) Analyzer() *analysis.Analyzer { @@ -94,6 +108,10 @@ func (t *TextField) Value() []byte { return t.value } +func (t *TextField) Text() string { + return string(t.value) +} + func (t *TextField) GoString() string { return fmt.Sprintf("&document.TextField{Name:%s, Options: %s, Analyzer: %v, Value: %s, ArrayPositions: %v}", t.name, t.options, t.analyzer, t.value, t.arrayPositions) } @@ -106,7 +124,7 @@ func NewTextField(name string, arrayPositions []uint64, value []byte) *TextField return NewTextFieldWithIndexingOptions(name, arrayPositions, value, DefaultTextIndexingOptions) } -func NewTextFieldWithIndexingOptions(name string, arrayPositions []uint64, value []byte, options IndexingOptions) *TextField { +func NewTextFieldWithIndexingOptions(name string, arrayPositions []uint64, value []byte, options index.FieldIndexingOptions) *TextField { return &TextField{ name: name, arrayPositions: arrayPositions, @@ -127,7 +145,7 @@ func NewTextFieldWithAnalyzer(name string, arrayPositions []uint64, value []byte } } -func NewTextFieldCustom(name string, arrayPositions []uint64, value []byte, options IndexingOptions, analyzer *analysis.Analyzer) *TextField { +func NewTextFieldCustom(name string, arrayPositions []uint64, value []byte, options index.FieldIndexingOptions, analyzer *analysis.Analyzer) *TextField { return &TextField{ name: name, arrayPositions: arrayPositions, diff --git a/vendor/github.com/blevesearch/bleve/error.go b/vendor/github.com/blevesearch/bleve/v2/error.go similarity index 95% rename from vendor/github.com/blevesearch/bleve/error.go rename to vendor/github.com/blevesearch/bleve/v2/error.go index 7402dfcf40..7dd21194c8 100644 --- a/vendor/github.com/blevesearch/bleve/error.go +++ b/vendor/github.com/blevesearch/bleve/v2/error.go @@ -20,7 +20,6 @@ const ( ErrorIndexPathDoesNotExist ErrorIndexMetaMissing ErrorIndexMetaCorrupt - ErrorUnknownStorageType ErrorIndexClosed ErrorAliasMulti ErrorAliasEmpty @@ -42,7 +41,6 @@ var errorMessages = map[Error]string{ ErrorIndexPathDoesNotExist: "cannot open index, path does not exist", ErrorIndexMetaMissing: "cannot open index, metadata missing", ErrorIndexMetaCorrupt: "cannot open index, metadata corrupt", - ErrorUnknownStorageType: "unknown storage type", ErrorIndexClosed: "index is closed", ErrorAliasMulti: "cannot perform single index operation on multiple index alias", ErrorAliasEmpty: "cannot perform operation on empty alias", diff --git a/vendor/github.com/blevesearch/bleve/geo/README.md b/vendor/github.com/blevesearch/bleve/v2/geo/README.md similarity index 100% rename from vendor/github.com/blevesearch/bleve/geo/README.md rename to vendor/github.com/blevesearch/bleve/v2/geo/README.md diff --git a/vendor/github.com/blevesearch/bleve/geo/geo.go b/vendor/github.com/blevesearch/bleve/v2/geo/geo.go similarity index 99% rename from vendor/github.com/blevesearch/bleve/geo/geo.go rename to vendor/github.com/blevesearch/bleve/v2/geo/geo.go index b18ace4337..55eace1df0 100644 --- a/vendor/github.com/blevesearch/bleve/geo/geo.go +++ b/vendor/github.com/blevesearch/bleve/v2/geo/geo.go @@ -18,7 +18,7 @@ import ( "fmt" "math" - "github.com/blevesearch/bleve/numeric" + "github.com/blevesearch/bleve/v2/numeric" ) // GeoBits is the number of bits used for a single geo point diff --git a/vendor/github.com/blevesearch/bleve/geo/geo_dist.go b/vendor/github.com/blevesearch/bleve/v2/geo/geo_dist.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/geo/geo_dist.go rename to vendor/github.com/blevesearch/bleve/v2/geo/geo_dist.go diff --git a/vendor/github.com/blevesearch/bleve/geo/geohash.go b/vendor/github.com/blevesearch/bleve/v2/geo/geohash.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/geo/geohash.go rename to vendor/github.com/blevesearch/bleve/v2/geo/geohash.go diff --git a/vendor/github.com/blevesearch/bleve/geo/parse.go b/vendor/github.com/blevesearch/bleve/v2/geo/parse.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/geo/parse.go rename to vendor/github.com/blevesearch/bleve/v2/geo/parse.go diff --git a/vendor/github.com/blevesearch/bleve/geo/sloppy.go b/vendor/github.com/blevesearch/bleve/v2/geo/sloppy.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/geo/sloppy.go rename to vendor/github.com/blevesearch/bleve/v2/geo/sloppy.go diff --git a/vendor/github.com/blevesearch/bleve/v2/go.mod b/vendor/github.com/blevesearch/bleve/v2/go.mod new file mode 100644 index 0000000000..1ee36d1aee --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/go.mod @@ -0,0 +1,29 @@ +module github.com/blevesearch/bleve/v2 + +go 1.13 + +require ( + github.com/RoaringBitmap/roaring v0.7.3 + github.com/bits-and-blooms/bitset v1.2.0 + github.com/blevesearch/bleve_index_api v1.0.1 + github.com/blevesearch/go-porterstemmer v1.0.3 + github.com/blevesearch/scorch_segment_api/v2 v2.0.1 + github.com/blevesearch/segment v0.9.0 + github.com/blevesearch/snowballstem v0.9.0 + github.com/blevesearch/upsidedown_store_api v1.0.1 + github.com/blevesearch/vellum v1.0.5 + github.com/blevesearch/zapx/v11 v11.2.2 + github.com/blevesearch/zapx/v12 v12.2.2 + github.com/blevesearch/zapx/v13 v13.2.2 + github.com/blevesearch/zapx/v14 v14.2.2 + github.com/blevesearch/zapx/v15 v15.2.2 + github.com/couchbase/moss v0.1.0 + github.com/golang/protobuf v1.3.2 + github.com/kljensen/snowball v0.6.0 + github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 + github.com/spf13/cobra v0.0.5 + github.com/steveyen/gtreap v0.1.0 + github.com/syndtr/goleveldb v1.0.0 + go.etcd.io/bbolt v1.3.5 + golang.org/x/text v0.3.0 +) diff --git a/vendor/github.com/blevesearch/bleve/v2/go.sum b/vendor/github.com/blevesearch/bleve/v2/go.sum new file mode 100644 index 0000000000..1087a75a8b --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/go.sum @@ -0,0 +1,152 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= +github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= +github.com/RoaringBitmap/roaring v0.7.3 h1:RwirWpvFONt2EwHHEHhER7S4BHZkyj3qL5LXLlnQPZ4= +github.com/RoaringBitmap/roaring v0.7.3/go.mod h1:jdT9ykXwHFNdJbEtxePexlFYH9LXucApeS0/+/g+p1I= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= +github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/blevesearch/bleve_index_api v1.0.0/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= +github.com/blevesearch/bleve_index_api v1.0.1 h1:nx9++0hnyiGOHJwQQYfsUGzpRdEVE5LsylmmngQvaFk= +github.com/blevesearch/bleve_index_api v1.0.1/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= +github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= +github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M= +github.com/blevesearch/mmap-go v1.0.2 h1:JtMHb+FgQCTTYIhtMvimw15dJwu1Y5lrZDMOFXVWPk0= +github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA= +github.com/blevesearch/scorch_segment_api/v2 v2.0.1 h1:fd+hPtZ8GsbqPK1HslGp7Vhoik4arZteA/IsCEgOisw= +github.com/blevesearch/scorch_segment_api/v2 v2.0.1/go.mod h1:lq7yK2jQy1yQjtjTfU931aVqz7pYxEudHaDwOt1tXfU= +github.com/blevesearch/segment v0.9.0 h1:5lG7yBCx98or7gK2cHMKPukPZ/31Kag7nONpoBt22Ac= +github.com/blevesearch/segment v0.9.0/go.mod h1:9PfHYUdQCgHktBgvtUOF4x+pc4/l8rdH0u5spnW85UQ= +github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s= +github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs= +github.com/blevesearch/upsidedown_store_api v1.0.1 h1:1SYRwyoFLwG3sj0ed89RLtM15amfX2pXlYbFOnF8zNU= +github.com/blevesearch/upsidedown_store_api v1.0.1/go.mod h1:MQDVGpHZrpe3Uy26zJBf/a8h0FZY6xJbthIMm8myH2Q= +github.com/blevesearch/vellum v1.0.5 h1:L5dJ7hKauRVbuH7I8uqLeSK92CPPY6FfrbAmLhAug8A= +github.com/blevesearch/vellum v1.0.5/go.mod h1:atE0EH3fvk43zzS7t1YNdNC7DbmcC3uz+eMD5xZ2OyQ= +github.com/blevesearch/zapx/v11 v11.2.2 h1:yeHRnGA4UPxVm1roONbp7VRm+SUx95AleE4rU8w4pc4= +github.com/blevesearch/zapx/v11 v11.2.2/go.mod h1:qunXXAB8awrvPgnHdqbPFMW01N93bhKFxkEWxYQgp8w= +github.com/blevesearch/zapx/v12 v12.2.2 h1:aK6r0DbMMI8+MnrqkmFwWxUr8ZwVwQf8owbdVv7uhKs= +github.com/blevesearch/zapx/v12 v12.2.2/go.mod h1:6reJkgolYR1r7GC6SwbuRGhvMWin+Ou/n2Cd7DdvYzY= +github.com/blevesearch/zapx/v13 v13.2.2 h1:6oa7kZhywrRT3CeuN0bIZC6EviQkbPOQpvdR5pnxCks= +github.com/blevesearch/zapx/v13 v13.2.2/go.mod h1:EeLDRSUIMxBFwD4hZxVioHLR7gpY4VF7kdUoImp4Vy8= +github.com/blevesearch/zapx/v14 v14.2.2 h1:lRZCBvQIByW8F+mCoY4uA6Uen2DCj7K+wCh5nVKNxew= +github.com/blevesearch/zapx/v14 v14.2.2/go.mod h1:zb01unR63/DfV2nvyAvgj64K8AKtj7WP6w23jtuHntQ= +github.com/blevesearch/zapx/v15 v15.2.2 h1:5+oWWAQTV3M0UNor05qrZujzxIXVuwtYU3ppZ7Y1aNI= +github.com/blevesearch/zapx/v15 v15.2.2/go.mod h1:I4QVJ432LKkZyNK1kZkh3OweKa+NSblZzIF0YSSExak= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/couchbase/ghistogram v0.1.0 h1:b95QcQTCzjTUocDXp/uMgSNQi8oj1tGwnJ4bODWZnps= +github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k= +github.com/couchbase/moss v0.1.0 h1:HCL+xxHUwmOaL44kMM/gU08OW6QGCui1WVFO58bjhNI= +github.com/couchbase/moss v0.1.0/go.mod h1:9MaHIaRuy9pvLPUJxB8sh8OrLfyDczECVL37grCIubs= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dvyukov/go-fuzz v0.0.0-20210429054444-fca39067bc72/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= +github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= +github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/kljensen/snowball v0.6.0 h1:6DZLCcZeL0cLfodx+Md4/OLC6b/bfurWUOUGs1ydfOU= +github.com/kljensen/snowball v0.6.0/go.mod h1:27N7E8fVU5H68RlUmnWwZCfxgt4POBJfENGMvNRhldw= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= +github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= +github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ= +github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc= +github.com/steveyen/gtreap v0.1.0 h1:CjhzTa274PyJLJuMZwIzCO1PfC00oRa8d1Kc78bFXJM= +github.com/steveyen/gtreap v0.1.0/go.mod h1:kl/5J7XbrOmlIbYIXdRHDDE5QxHqpk0cmkT7Z4dM9/Y= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= +github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= +go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200928182047-19e03678916f/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/vendor/github.com/blevesearch/bleve/index.go b/vendor/github.com/blevesearch/bleve/v2/index.go similarity index 92% rename from vendor/github.com/blevesearch/bleve/index.go rename to vendor/github.com/blevesearch/bleve/v2/index.go index 974358b81e..e297122fac 100644 --- a/vendor/github.com/blevesearch/bleve/index.go +++ b/vendor/github.com/blevesearch/bleve/v2/index.go @@ -17,11 +17,12 @@ package bleve import ( "context" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/index/upsidedown" + + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) // A Batch groups together multiple Index and Delete @@ -71,7 +72,7 @@ func (b *Batch) TotalDocsSize() uint64 { // batch which skips the mapping. NOTE: the bleve Index is not updated // until the batch is executed. func (b *Batch) IndexAdvanced(doc *document.Document) (err error) { - if doc.ID == "" { + if doc.ID() == "" { return ErrorEmptyID } b.internal.Update(doc) @@ -216,7 +217,7 @@ type Index interface { // Document returns specified document or nil if the document is not // indexed or stored. - Document(id string) (*document.Document, error) + Document(id string) (index.Document, error) // DocCount returns the number of documents in the index. DocCount() (uint64, error) @@ -245,9 +246,8 @@ type Index interface { // SetName lets you assign your own logical name to this index SetName(string) - // Advanced returns the indexer and data store, exposing lower level - // methods to enumerate records and access data. - Advanced() (index.Index, store.KVStore, error) + // Advanced returns the internal index implementation + Advanced() (index.Index, error) } // New index at the specified path, must not exist. @@ -263,7 +263,7 @@ func New(path string, mapping mapping.IndexMapping) (Index, error) { // The provided mapping will be used for all // Index/Search operations. func NewMemOnly(mapping mapping.IndexMapping) (Index, error) { - return newIndexUsing("", mapping, Config.DefaultIndexType, Config.DefaultMemKVStore, nil) + return newIndexUsing("", mapping, upsidedown.Name, Config.DefaultMemKVStore, nil) } // NewUsing creates index at the specified path, @@ -307,3 +307,15 @@ type Builder interface { func NewBuilder(path string, mapping mapping.IndexMapping, config map[string]interface{}) (Builder, error) { return newBuilder(path, mapping, config) } + +// IndexCopyable is an index which supports an online copy operation +// of the index. +type IndexCopyable interface { + // CopyTo creates a fully functional copy of the index at the + // specified destination directory implementation. + CopyTo(d index.Directory) error +} + +// FileSystemDirectory is the default implementation for the +// index.Directory interface. +type FileSystemDirectory string diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/README.md b/vendor/github.com/blevesearch/bleve/v2/index/scorch/README.md similarity index 100% rename from vendor/github.com/blevesearch/bleve/index/scorch/README.md rename to vendor/github.com/blevesearch/bleve/v2/index/scorch/README.md diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/builder.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/builder.go similarity index 93% rename from vendor/github.com/blevesearch/bleve/index/scorch/builder.go rename to vendor/github.com/blevesearch/bleve/v2/index/scorch/builder.go index 1f4b41d639..5e64230c51 100644 --- a/vendor/github.com/blevesearch/bleve/index/scorch/builder.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/builder.go @@ -21,9 +21,8 @@ import ( "sync" "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" bolt "go.etcd.io/bbolt" ) @@ -40,7 +39,7 @@ type Builder struct { mergeMax int batch *index.Batch internal map[string][]byte - segPlugin segment.Plugin + segPlugin SegmentPlugin } func NewBuilder(config map[string]interface{}) (*Builder, error) { @@ -117,7 +116,7 @@ func (o *Builder) parseConfig(config map[string]interface{}) (err error) { // Index will place the document into the index. // It is invalid to index the same document multiple times. -func (o *Builder) Index(doc *document.Document) error { +func (o *Builder) Index(doc index.Document) error { o.m.Lock() defer o.m.Unlock() @@ -135,14 +134,14 @@ func (o *Builder) maybeFlushBatchLOCKED(moreThan int) error { } func (o *Builder) executeBatchLOCKED(batch *index.Batch) (err error) { - analysisResults := make([]*index.AnalysisResult, 0, len(batch.IndexOps)) + analysisResults := make([]index.Document, 0, len(batch.IndexOps)) for _, doc := range batch.IndexOps { if doc != nil { // insert _id field - doc.AddField(document.NewTextFieldCustom("_id", nil, []byte(doc.ID), document.IndexField|document.StoreField, nil)) + doc.AddIDField() // perform analysis directly - analysisResult := analyze(doc) - analysisResults = append(analysisResults, analysisResult) + analyze(doc) + analysisResults = append(analysisResults, doc) } } @@ -305,7 +304,7 @@ func (o *Builder) Close() error { } // fill the root bolt with this fake index snapshot - _, _, err = prepareBoltSnapshot(is, tx, o.path, o.segPlugin) + _, _, err = prepareBoltSnapshot(is, tx, o.path, o.segPlugin, nil) if err != nil { _ = tx.Rollback() _ = rootBolt.Close() diff --git a/vendor/github.com/blevesearch/bleve/v2/index/scorch/empty.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/empty.go new file mode 100644 index 0000000000..4d47555315 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/empty.go @@ -0,0 +1,33 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import segment "github.com/blevesearch/scorch_segment_api/v2" + +type emptyPostingsIterator struct{} + +func (e *emptyPostingsIterator) Next() (segment.Posting, error) { + return nil, nil +} + +func (e *emptyPostingsIterator) Advance(uint64) (segment.Posting, error) { + return nil, nil +} + +func (e *emptyPostingsIterator) Size() int { + return 0 +} + +var anEmptyPostingsIterator = &emptyPostingsIterator{} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/event.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/event.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/index/scorch/event.go rename to vendor/github.com/blevesearch/bleve/v2/index/scorch/event.go diff --git a/vendor/github.com/blevesearch/bleve/v2/index/scorch/int.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/int.go new file mode 100644 index 0000000000..4fa6d7f71f --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/int.go @@ -0,0 +1,92 @@ +// Copyright 2014 The Cockroach Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +// This code originated from: +// https://github.com/cockroachdb/cockroach/blob/2dd65dde5d90c157f4b93f92502ca1063b904e1d/pkg/util/encoding/encoding.go + +// Modified to not use pkg/errors + +package scorch + +import "fmt" + +const ( + // intMin is chosen such that the range of int tags does not overlap the + // ascii character set that is frequently used in testing. + intMin = 0x80 // 128 + intMaxWidth = 8 + intZero = intMin + intMaxWidth // 136 + intSmall = intMax - intZero - intMaxWidth // 109 + // intMax is the maximum int tag value. + intMax = 0xfd // 253 +) + +// encodeUvarintAscending encodes the uint64 value using a variable length +// (length-prefixed) representation. The length is encoded as a single +// byte indicating the number of encoded bytes (-8) to follow. See +// EncodeVarintAscending for rationale. The encoded bytes are appended to the +// supplied buffer and the final buffer is returned. +func encodeUvarintAscending(b []byte, v uint64) []byte { + switch { + case v <= intSmall: + return append(b, intZero+byte(v)) + case v <= 0xff: + return append(b, intMax-7, byte(v)) + case v <= 0xffff: + return append(b, intMax-6, byte(v>>8), byte(v)) + case v <= 0xffffff: + return append(b, intMax-5, byte(v>>16), byte(v>>8), byte(v)) + case v <= 0xffffffff: + return append(b, intMax-4, byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) + case v <= 0xffffffffff: + return append(b, intMax-3, byte(v>>32), byte(v>>24), byte(v>>16), byte(v>>8), + byte(v)) + case v <= 0xffffffffffff: + return append(b, intMax-2, byte(v>>40), byte(v>>32), byte(v>>24), byte(v>>16), + byte(v>>8), byte(v)) + case v <= 0xffffffffffffff: + return append(b, intMax-1, byte(v>>48), byte(v>>40), byte(v>>32), byte(v>>24), + byte(v>>16), byte(v>>8), byte(v)) + default: + return append(b, intMax, byte(v>>56), byte(v>>48), byte(v>>40), byte(v>>32), + byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) + } +} + +// decodeUvarintAscending decodes a varint encoded uint64 from the input +// buffer. The remainder of the input buffer and the decoded uint64 +// are returned. +func decodeUvarintAscending(b []byte) ([]byte, uint64, error) { + if len(b) == 0 { + return nil, 0, fmt.Errorf("insufficient bytes to decode uvarint value") + } + length := int(b[0]) - intZero + b = b[1:] // skip length byte + if length <= intSmall { + return b, uint64(length), nil + } + length -= intSmall + if length < 0 || length > 8 { + return nil, 0, fmt.Errorf("invalid uvarint length of %d", length) + } else if len(b) < length { + return nil, 0, fmt.Errorf("insufficient bytes to decode uvarint value: %q", b) + } + var v uint64 + // It is faster to range over the elements in a slice than to index + // into the slice on each loop iteration. + for _, t := range b[:length] { + v = (v << 8) | uint64(t) + } + return b[length:], v, nil +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/introducer.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/introducer.go similarity index 99% rename from vendor/github.com/blevesearch/bleve/index/scorch/introducer.go rename to vendor/github.com/blevesearch/bleve/v2/index/scorch/introducer.go index 7770c41c52..8516d411db 100644 --- a/vendor/github.com/blevesearch/bleve/index/scorch/introducer.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/introducer.go @@ -19,8 +19,8 @@ import ( "sync/atomic" "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" ) type segmentIntroduction struct { diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/merge.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/merge.go similarity index 97% rename from vendor/github.com/blevesearch/bleve/index/scorch/merge.go rename to vendor/github.com/blevesearch/bleve/v2/index/scorch/merge.go index 56c0953f46..30f7a4f23e 100644 --- a/vendor/github.com/blevesearch/bleve/index/scorch/merge.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/merge.go @@ -24,8 +24,8 @@ import ( "time" "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index/scorch/mergeplan" - "github.com/blevesearch/bleve/index/scorch/segment" + "github.com/blevesearch/bleve/v2/index/scorch/mergeplan" + segment "github.com/blevesearch/scorch_segment_api/v2" ) func (s *Scorch) mergerLoop() { @@ -209,32 +209,32 @@ func (s *Scorch) parseMergePlannerOptions() (*mergeplan.MergePlanOptions, } type closeChWrapper struct { - ch1 chan struct{} - ctx context.Context - closeCh chan struct{} + ch1 chan struct{} + ctx context.Context + closeCh chan struct{} + cancelCh chan struct{} } func newCloseChWrapper(ch1 chan struct{}, ctx context.Context) *closeChWrapper { - return &closeChWrapper{ch1: ch1, - ctx: ctx, - closeCh: make(chan struct{})} + return &closeChWrapper{ + ch1: ch1, + ctx: ctx, + closeCh: make(chan struct{}), + cancelCh: make(chan struct{}), + } } func (w *closeChWrapper) close() { - select { - case <-w.closeCh: - default: - close(w.closeCh) - } + close(w.closeCh) } func (w *closeChWrapper) listen() { select { case <-w.ch1: - w.close() + close(w.cancelCh) case <-w.ctx.Done(): - w.close() + close(w.cancelCh) case <-w.closeCh: } } @@ -320,7 +320,7 @@ func (s *Scorch) planMergeAtSnapshot(ctx context.Context, atomic.AddUint64(&s.stats.TotFileMergeZapBeg, 1) newDocNums, _, err := s.segPlugin.Merge(segmentsToMerge, docsToDrop, path, - cw.closeCh, s) + cw.cancelCh, s) atomic.AddUint64(&s.stats.TotFileMergeZapEnd, 1) fileMergeZapTime := uint64(time.Since(fileMergeZapStartTime)) diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/mergeplan/merge_plan.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/mergeplan/merge_plan.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/index/scorch/mergeplan/merge_plan.go rename to vendor/github.com/blevesearch/bleve/v2/index/scorch/mergeplan/merge_plan.go diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/mergeplan/sort.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/mergeplan/sort.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/index/scorch/mergeplan/sort.go rename to vendor/github.com/blevesearch/bleve/v2/index/scorch/mergeplan/sort.go diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/optimize.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/optimize.go similarity index 93% rename from vendor/github.com/blevesearch/bleve/index/scorch/optimize.go rename to vendor/github.com/blevesearch/bleve/v2/index/scorch/optimize.go index 658354cd71..3c7969fa9e 100644 --- a/vendor/github.com/blevesearch/bleve/index/scorch/optimize.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/optimize.go @@ -17,8 +17,8 @@ package scorch import ( "fmt" "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" "sync/atomic" ) @@ -174,9 +174,9 @@ OUTER: var docNum1HitLastOk bool for _, tfr := range o.tfrs { - if _, ok := tfr.iterators[i].(*segment.EmptyPostingsIterator); ok { + if _, ok := tfr.iterators[i].(*emptyPostingsIterator); ok { // An empty postings iterator means the entire AND is empty. - oTFR.iterators[i] = segment.AnEmptyPostingsIterator + oTFR.iterators[i] = anEmptyPostingsIterator continue OUTER } @@ -193,7 +193,7 @@ OUTER: if docNum1HitLastOk && docNum1HitLast != docNum1Hit { // The docNum1Hit doesn't match the previous // docNum1HitLast, so the entire AND is empty. - oTFR.iterators[i] = segment.AnEmptyPostingsIterator + oTFR.iterators[i] = anEmptyPostingsIterator continue OUTER } @@ -205,7 +205,7 @@ OUTER: if itr.ActualBitmap() == nil { // An empty actual bitmap means the entire AND is empty. - oTFR.iterators[i] = segment.AnEmptyPostingsIterator + oTFR.iterators[i] = anEmptyPostingsIterator continue OUTER } @@ -221,14 +221,14 @@ OUTER: if !bm.Contains(uint32(docNum1HitLast)) { // The docNum1Hit isn't in one of our actual // bitmaps, so the entire AND is empty. - oTFR.iterators[i] = segment.AnEmptyPostingsIterator + oTFR.iterators[i] = anEmptyPostingsIterator continue OUTER } } // The actual bitmaps and docNum1Hits all contain or have // the same 1-hit docNum, so that's our AND'ed result. - oTFR.iterators[i] = segment.NewUnadornedPostingsIteratorFrom1Hit(docNum1HitLast) + oTFR.iterators[i] = newUnadornedPostingsIteratorFrom1Hit(docNum1HitLast) continue OUTER } @@ -236,13 +236,13 @@ OUTER: if len(actualBMs) == 0 { // If we've collected no actual bitmaps at this point, // then the entire AND is empty. - oTFR.iterators[i] = segment.AnEmptyPostingsIterator + oTFR.iterators[i] = anEmptyPostingsIterator continue OUTER } if len(actualBMs) == 1 { // If we've only 1 actual bitmap, then that's our result. - oTFR.iterators[i] = segment.NewUnadornedPostingsIteratorFromBitmap(actualBMs[0]) + oTFR.iterators[i] = newUnadornedPostingsIteratorFromBitmap(actualBMs[0]) continue OUTER } @@ -254,7 +254,7 @@ OUTER: bm.And(actualBM) } - oTFR.iterators[i] = segment.NewUnadornedPostingsIteratorFromBitmap(bm) + oTFR.iterators[i] = newUnadornedPostingsIteratorFromBitmap(bm) } atomic.AddUint64(&o.snapshot.parent.stats.TotTermSearchersStarted, uint64(1)) @@ -369,7 +369,7 @@ func (o *OptimizeTFRDisjunctionUnadorned) Finish() (rv index.Optimized, err erro bm.AddMany(docNums) - oTFR.iterators[i] = segment.NewUnadornedPostingsIteratorFromBitmap(bm) + oTFR.iterators[i] = newUnadornedPostingsIteratorFromBitmap(bm) } atomic.AddUint64(&o.snapshot.parent.stats.TotTermSearchersStarted, uint64(1)) diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/persister.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/persister.go similarity index 93% rename from vendor/github.com/blevesearch/bleve/index/scorch/persister.go rename to vendor/github.com/blevesearch/bleve/v2/index/scorch/persister.go index 498378a4f8..1688edaede 100644 --- a/vendor/github.com/blevesearch/bleve/index/scorch/persister.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/persister.go @@ -19,6 +19,7 @@ import ( "encoding/binary" "encoding/json" "fmt" + "io" "io/ioutil" "log" "math" @@ -30,8 +31,8 @@ import ( "time" "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" bolt "go.etcd.io/bbolt" ) @@ -93,7 +94,6 @@ func (s *Scorch) persisterLoop() { po, err := s.parsePersisterOptions() if err != nil { s.fireAsyncError(fmt.Errorf("persisterOptions json parsing err: %v", err)) - s.asyncTasks.Done() return } @@ -428,13 +428,64 @@ func (s *Scorch) persistSnapshotMaybeMerge(snapshot *IndexSnapshot) ( return true, nil } +func copyToDirectory(srcPath string, d index.Directory) (int64, error) { + if d == nil { + return 0, nil + } + + dest, err := d.GetWriter(filepath.Join("store", filepath.Base(srcPath))) + if err != nil { + return 0, fmt.Errorf("GetWriter err: %v", err) + } + + sourceFileStat, err := os.Stat(srcPath) + if err != nil { + return 0, err + } + + if !sourceFileStat.Mode().IsRegular() { + return 0, fmt.Errorf("%s is not a regular file", srcPath) + } + + source, err := os.Open(srcPath) + if err != nil { + return 0, err + } + defer source.Close() + defer dest.Close() + return io.Copy(dest, source) +} + +func persistToDirectory(seg segment.UnpersistedSegment, d index.Directory, + path string) error { + if d == nil { + return seg.Persist(path) + } + + sg, ok := seg.(io.WriterTo) + if !ok { + return fmt.Errorf("no io.WriterTo segment implementation found") + } + + w, err := d.GetWriter(filepath.Join("store", filepath.Base(path))) + if err != nil { + return err + } + + _, err = sg.WriteTo(w) + w.Close() + + return err +} + func prepareBoltSnapshot(snapshot *IndexSnapshot, tx *bolt.Tx, path string, - segPlugin segment.Plugin) ([]string, map[uint64]string, error) { + segPlugin SegmentPlugin, d index.Directory) ( + []string, map[uint64]string, error) { snapshotsBucket, err := tx.CreateBucketIfNotExists(boltSnapshotsBucket) if err != nil { return nil, nil, err } - newSnapshotKey := segment.EncodeUvarintAscending(nil, snapshot.epoch) + newSnapshotKey := encodeUvarintAscending(nil, snapshot.epoch) snapshotBucket, err := snapshotsBucket.CreateBucketIfNotExists(newSnapshotKey) if err != nil { return nil, nil, err @@ -474,7 +525,7 @@ func prepareBoltSnapshot(snapshot *IndexSnapshot, tx *bolt.Tx, path string, // first ensure that each segment in this snapshot has been persisted for _, segmentSnapshot := range snapshot.segment { - snapshotSegmentKey := segment.EncodeUvarintAscending(nil, segmentSnapshot.id) + snapshotSegmentKey := encodeUvarintAscending(nil, segmentSnapshot.id) snapshotSegmentBucket, err := snapshotBucket.CreateBucketIfNotExists(snapshotSegmentKey) if err != nil { return nil, nil, err @@ -482,7 +533,11 @@ func prepareBoltSnapshot(snapshot *IndexSnapshot, tx *bolt.Tx, path string, switch seg := segmentSnapshot.segment.(type) { case segment.PersistedSegment: segPath := seg.Path() - filename := strings.TrimPrefix(segPath, path+string(os.PathSeparator)) + _, err = copyToDirectory(segPath, d) + if err != nil { + return nil, nil, fmt.Errorf("segment: %s copy err: %v", segPath, err) + } + filename := filepath.Base(segPath) err = snapshotSegmentBucket.Put(boltPathKey, []byte(filename)) if err != nil { return nil, nil, err @@ -491,10 +546,10 @@ func prepareBoltSnapshot(snapshot *IndexSnapshot, tx *bolt.Tx, path string, case segment.UnpersistedSegment: // need to persist this to disk filename := zapFileName(segmentSnapshot.id) - path := path + string(os.PathSeparator) + filename - err = seg.Persist(path) + path := filepath.Join(path, filename) + err := persistToDirectory(seg, d, path) if err != nil { - return nil, nil, fmt.Errorf("error persisting segment: %v", err) + return nil, nil, fmt.Errorf("segment: %s persist err: %v", path, err) } newSegmentPaths[segmentSnapshot.id] = path err = snapshotSegmentBucket.Put(boltPathKey, []byte(filename)) @@ -535,7 +590,7 @@ func (s *Scorch) persistSnapshotDirect(snapshot *IndexSnapshot) (err error) { } }() - filenames, newSegmentPaths, err := prepareBoltSnapshot(snapshot, tx, s.path, s.segPlugin) + filenames, newSegmentPaths, err := prepareBoltSnapshot(snapshot, tx, s.path, s.segPlugin, nil) if err != nil { return err } @@ -628,7 +683,7 @@ func (s *Scorch) loadFromBolt() error { foundRoot := false c := snapshots.Cursor() for k, _ := c.Last(); k != nil; k, _ = c.Prev() { - _, snapshotEpoch, err := segment.DecodeUvarintAscending(k) + _, snapshotEpoch, err := decodeUvarintAscending(k) if err != nil { log.Printf("unable to parse segment epoch %x, continuing", k) continue @@ -680,7 +735,7 @@ func (s *Scorch) LoadSnapshot(epoch uint64) (rv *IndexSnapshot, err error) { if snapshots == nil { return nil } - snapshotKey := segment.EncodeUvarintAscending(nil, epoch) + snapshotKey := encodeUvarintAscending(nil, epoch) snapshot := snapshots.Bucket(snapshotKey) if snapshot == nil { return fmt.Errorf("snapshot with epoch: %v - doesn't exist", epoch) @@ -744,7 +799,7 @@ func (s *Scorch) loadSnapshot(snapshot *bolt.Bucket) (*IndexSnapshot, error) { _ = rv.DecRef() return nil, fmt.Errorf("failed to load segment: %v", err) } - _, segmentSnapshot.id, err = segment.DecodeUvarintAscending(k) + _, segmentSnapshot.id, err = decodeUvarintAscending(k) if err != nil { _ = rv.DecRef() return nil, fmt.Errorf("failed to decode segment id: %v", err) @@ -865,7 +920,7 @@ func (s *Scorch) removeOldBoltSnapshots() (numRemoved int, err error) { } for _, epochToRemove := range epochsToRemove { - k := segment.EncodeUvarintAscending(nil, epochToRemove) + k := encodeUvarintAscending(nil, epochToRemove) err = snapshots.DeleteBucket(k) if err == bolt.ErrBucketNotFound { err = nil @@ -941,7 +996,7 @@ func (s *Scorch) RootBoltSnapshotEpochs() ([]uint64, error) { } sc := snapshots.Cursor() for sk, _ := sc.Last(); sk != nil; sk, _ = sc.Prev() { - _, snapshotEpoch, err := segment.DecodeUvarintAscending(sk) + _, snapshotEpoch, err := decodeUvarintAscending(sk) if err != nil { continue } diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/segment/regexp.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/regexp.go similarity index 73% rename from vendor/github.com/blevesearch/bleve/index/scorch/segment/regexp.go rename to vendor/github.com/blevesearch/bleve/v2/index/scorch/regexp.go index 3a31f41498..5a3584f51a 100644 --- a/vendor/github.com/blevesearch/bleve/index/scorch/segment/regexp.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/regexp.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018 Couchbase, Inc. +// Copyright (c) 2020 Couchbase, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,15 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package segment +package scorch import ( "regexp/syntax" - "github.com/couchbase/vellum/regexp" + "github.com/blevesearch/vellum/regexp" ) -func ParseRegexp(pattern string) (a *regexp.Regexp, prefixBeg, prefixEnd []byte, err error) { +func parseRegexp(pattern string) (a *regexp.Regexp, prefixBeg, prefixEnd []byte, err error) { // TODO: potential optimization where syntax.Regexp supports a Simplify() API? parsed, err := syntax.Parse(pattern, syntax.Perl) @@ -33,10 +33,10 @@ func ParseRegexp(pattern string) (a *regexp.Regexp, prefixBeg, prefixEnd []byte, return nil, nil, nil, err } - prefix := LiteralPrefix(parsed) + prefix := literalPrefix(parsed) if prefix != "" { prefixBeg := []byte(prefix) - prefixEnd := IncrementBytes(prefixBeg) + prefixEnd := calculateExclusiveEndFromPrefix(prefixBeg) return re, prefixBeg, prefixEnd, nil } @@ -44,7 +44,7 @@ func ParseRegexp(pattern string) (a *regexp.Regexp, prefixBeg, prefixEnd []byte, } // Returns the literal prefix given the parse tree for a regexp -func LiteralPrefix(s *syntax.Regexp) string { +func literalPrefix(s *syntax.Regexp) string { // traverse the left-most branch in the parse tree as long as the // node represents a concatenation for s != nil && s.Op == syntax.OpConcat { @@ -61,15 +61,3 @@ func LiteralPrefix(s *syntax.Regexp) string { return "" // no literal prefix } - -func IncrementBytes(in []byte) []byte { - rv := make([]byte, len(in)) - copy(rv, in) - for i := len(rv) - 1; i >= 0; i-- { - rv[i] = rv[i] + 1 - if rv[i] != 0 { - return rv // didn't overflow, so stop - } - } - return nil // overflowed -} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/rollback.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/rollback.go similarity index 95% rename from vendor/github.com/blevesearch/bleve/index/scorch/rollback.go rename to vendor/github.com/blevesearch/bleve/v2/index/scorch/rollback.go index 7cc87bdea0..067220e6ff 100644 --- a/vendor/github.com/blevesearch/bleve/index/scorch/rollback.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/rollback.go @@ -19,7 +19,6 @@ import ( "log" "os" - "github.com/blevesearch/bleve/index/scorch/segment" bolt "go.etcd.io/bbolt" ) @@ -71,7 +70,7 @@ func RollbackPoints(path string) ([]*RollbackPoint, error) { c1 := snapshots.Cursor() for k, _ := c1.Last(); k != nil; k, _ = c1.Prev() { - _, snapshotEpoch, err := segment.DecodeUvarintAscending(k) + _, snapshotEpoch, err := decodeUvarintAscending(k) if err != nil { log.Printf("RollbackPoints:"+ " unable to parse segment epoch %x, continuing", k) @@ -154,7 +153,7 @@ func Rollback(path string, to *RollbackPoint) error { } sc := snapshots.Cursor() for sk, _ := sc.Last(); sk != nil && !found; sk, _ = sc.Prev() { - _, snapshotEpoch, err := segment.DecodeUvarintAscending(sk) + _, snapshotEpoch, err := decodeUvarintAscending(sk) if err != nil { continue } @@ -195,7 +194,7 @@ func Rollback(path string, to *RollbackPoint) error { return nil } for _, epoch := range eligibleEpochs { - k := segment.EncodeUvarintAscending(nil, epoch) + k := encodeUvarintAscending(nil, epoch) if err != nil { continue } diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/scorch.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/scorch.go similarity index 91% rename from vendor/github.com/blevesearch/bleve/index/scorch/scorch.go rename to vendor/github.com/blevesearch/bleve/v2/index/scorch/scorch.go index fccff67ab5..fa1aaebbfe 100644 --- a/vendor/github.com/blevesearch/bleve/index/scorch/scorch.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/scorch.go @@ -24,12 +24,9 @@ import ( "time" "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/registry" + "github.com/blevesearch/bleve/v2/registry" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" bolt "go.etcd.io/bbolt" ) @@ -75,7 +72,7 @@ type Scorch struct { forceMergeRequestCh chan *mergerCtrl - segPlugin segment.Plugin + segPlugin SegmentPlugin } type internalStats struct { @@ -202,11 +199,9 @@ func (s *Scorch) openBolt() error { s.unsafeBatch = true } - var rootBoltOpt *bolt.Options + var rootBoltOpt = *bolt.DefaultOptions if s.readOnly { - rootBoltOpt = &bolt.Options{ - ReadOnly: true, - } + rootBoltOpt.ReadOnly = true } else { if s.path != "" { err := os.MkdirAll(s.path, 0700) @@ -216,10 +211,19 @@ func (s *Scorch) openBolt() error { } } + if boltTimeoutStr, ok := s.config["bolt_timeout"].(string); ok { + var err error + boltTimeout, err := time.ParseDuration(boltTimeoutStr) + if err != nil { + return fmt.Errorf("invalid duration specified for bolt_timeout: %v", err) + } + rootBoltOpt.Timeout = boltTimeout + } + rootBoltPath := s.path + string(os.PathSeparator) + "root.bolt" var err error if s.path != "" { - s.rootBolt, err = bolt.Open(rootBoltPath, 0600, rootBoltOpt) + s.rootBolt, err = bolt.Open(rootBoltPath, 0600, &rootBoltOpt) if err != nil { return err } @@ -293,7 +297,7 @@ func (s *Scorch) Close() (err error) { return } -func (s *Scorch) Update(doc *document.Document) error { +func (s *Scorch) Update(doc index.Document) error { b := index.NewBatch() b.Update(doc) return s.Batch(b) @@ -313,7 +317,7 @@ func (s *Scorch) Batch(batch *index.Batch) (err error) { s.fireEvent(EventKindBatchIntroduction, time.Since(start)) }() - resultChan := make(chan *index.AnalysisResult, len(batch.IndexOps)) + resultChan := make(chan index.Document, len(batch.IndexOps)) var numUpdates uint64 var numDeletes uint64 @@ -322,7 +326,7 @@ func (s *Scorch) Batch(batch *index.Batch) (err error) { for docID, doc := range batch.IndexOps { if doc != nil { // insert _id field - doc.AddField(document.NewTextFieldCustom("_id", nil, []byte(doc.ID), document.IndexField|document.StoreField, nil)) + doc.AddIDField() numUpdates++ numPlainTextBytes += doc.NumPlainTextBytes() } else { @@ -335,18 +339,21 @@ func (s *Scorch) Batch(batch *index.Batch) (err error) { if numUpdates > 0 { go func() { - for _, doc := range batch.IndexOps { + for k := range batch.IndexOps { + doc := batch.IndexOps[k] if doc != nil { - aw := index.NewAnalysisWork(s, doc, resultChan) // put the work on the queue - s.analysisQueue.Queue(aw) + s.analysisQueue.Queue(func() { + analyze(doc) + resultChan <- doc + }) } } }() } // wait for analysis result - analysisResults := make([]*index.AnalysisResult, int(numUpdates)) + analysisResults := make([]index.Document, int(numUpdates)) var itemsDeQueued uint64 var totalAnalysisSize int for itemsDeQueued < numUpdates { @@ -566,37 +573,23 @@ func (s *Scorch) StatsMap() map[string]interface{} { return m } -func (s *Scorch) Analyze(d *document.Document) *index.AnalysisResult { - return analyze(d) +func (s *Scorch) Analyze(d index.Document) { + analyze(d) } -func analyze(d *document.Document) *index.AnalysisResult { - rv := &index.AnalysisResult{ - Document: d, - Analyzed: make([]analysis.TokenFrequencies, len(d.Fields)+len(d.CompositeFields)), - Length: make([]int, len(d.Fields)+len(d.CompositeFields)), - } - - for i, field := range d.Fields { +func analyze(d index.Document) { + d.VisitFields(func(field index.Field) { if field.Options().IsIndexed() { - fieldLength, tokenFreqs := field.Analyze() - rv.Analyzed[i] = tokenFreqs - rv.Length[i] = fieldLength + field.Analyze() - if len(d.CompositeFields) > 0 && field.Name() != "_id" { + if d.HasComposite() && field.Name() != "_id" { // see if any of the composite fields need this - for _, compositeField := range d.CompositeFields { - compositeField.Compose(field.Name(), fieldLength, tokenFreqs) - } + d.VisitComposite(func(cf index.CompositeField) { + cf.Compose(field.Name(), field.AnalyzedLength(), field.AnalyzedTokenFrequencies()) + }) } } - } - - return rv -} - -func (s *Scorch) Advanced() (store.KVStore, error) { - return nil, nil + }) } func (s *Scorch) AddEligibleForRemoval(epoch uint64) { diff --git a/vendor/github.com/blevesearch/bleve/v2/index/scorch/segment_plugin.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/segment_plugin.go new file mode 100644 index 0000000000..ea40911fa9 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/segment_plugin.go @@ -0,0 +1,133 @@ +// Copyright (c) 2019 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "fmt" + "github.com/RoaringBitmap/roaring" + index "github.com/blevesearch/bleve_index_api" + + segment "github.com/blevesearch/scorch_segment_api/v2" + + zapv11 "github.com/blevesearch/zapx/v11" + zapv12 "github.com/blevesearch/zapx/v12" + zapv13 "github.com/blevesearch/zapx/v13" + zapv14 "github.com/blevesearch/zapx/v14" + zapv15 "github.com/blevesearch/zapx/v15" +) + +// SegmentPlugin represents the essential functions required by a package to plug in +// it's segment implementation +type SegmentPlugin interface { + + // Type is the name for this segment plugin + Type() string + + // Version is a numeric value identifying a specific version of this type. + // When incompatible changes are made to a particular type of plugin, the + // version must be incremented. + Version() uint32 + + // New takes a set of Documents and turns them into a new Segment + New(results []index.Document) (segment.Segment, uint64, error) + + // Open attempts to open the file at the specified path and + // return the corresponding Segment + Open(path string) (segment.Segment, error) + + // Merge takes a set of Segments, and creates a new segment on disk at + // the specified path. + // Drops is a set of bitmaps (one for each segment) indicating which + // documents can be dropped from the segments during the merge. + // If the closeCh channel is closed, Merge will cease doing work at + // the next opportunity, and return an error (closed). + // StatsReporter can optionally be provided, in which case progress + // made during the merge is reported while operation continues. + // Returns: + // A slice of new document numbers (one for each input segment), + // this allows the caller to know a particular document's new + // document number in the newly merged segment. + // The number of bytes written to the new segment file. + // An error, if any occurred. + Merge(segments []segment.Segment, drops []*roaring.Bitmap, path string, + closeCh chan struct{}, s segment.StatsReporter) ( + [][]uint64, uint64, error) +} + +var supportedSegmentPlugins map[string]map[uint32]SegmentPlugin +var defaultSegmentPlugin SegmentPlugin + +func init() { + ResetSegmentPlugins() + RegisterSegmentPlugin(&zapv15.ZapPlugin{}, true) + RegisterSegmentPlugin(&zapv14.ZapPlugin{}, false) + RegisterSegmentPlugin(&zapv13.ZapPlugin{}, false) + RegisterSegmentPlugin(&zapv12.ZapPlugin{}, false) + RegisterSegmentPlugin(&zapv11.ZapPlugin{}, false) +} + +func ResetSegmentPlugins() { + supportedSegmentPlugins = map[string]map[uint32]SegmentPlugin{} +} + +func RegisterSegmentPlugin(plugin SegmentPlugin, makeDefault bool) { + if _, ok := supportedSegmentPlugins[plugin.Type()]; !ok { + supportedSegmentPlugins[plugin.Type()] = map[uint32]SegmentPlugin{} + } + supportedSegmentPlugins[plugin.Type()][plugin.Version()] = plugin + if makeDefault { + defaultSegmentPlugin = plugin + } +} + +func SupportedSegmentTypes() (rv []string) { + for k := range supportedSegmentPlugins { + rv = append(rv, k) + } + return +} + +func SupportedSegmentTypeVersions(typ string) (rv []uint32) { + for k := range supportedSegmentPlugins[typ] { + rv = append(rv, k) + } + return rv +} + +func chooseSegmentPlugin(forcedSegmentType string, + forcedSegmentVersion uint32) (SegmentPlugin, error) { + if versions, ok := supportedSegmentPlugins[forcedSegmentType]; ok { + if segPlugin, ok := versions[uint32(forcedSegmentVersion)]; ok { + return segPlugin, nil + } + return nil, fmt.Errorf( + "unsupported version %d for segment type: %s, supported: %v", + forcedSegmentVersion, forcedSegmentType, + SupportedSegmentTypeVersions(forcedSegmentType)) + } + return nil, fmt.Errorf("unsupported segment type: %s, supported: %v", + forcedSegmentType, SupportedSegmentTypes()) +} + +func (s *Scorch) loadSegmentPlugin(forcedSegmentType string, + forcedSegmentVersion uint32) error { + segPlugin, err := chooseSegmentPlugin(forcedSegmentType, + forcedSegmentVersion) + if err != nil { + return err + } + s.segPlugin = segPlugin + return nil +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index.go similarity index 83% rename from vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index.go rename to vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index.go index 61204ebbca..98282eae66 100644 --- a/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index.go @@ -18,17 +18,20 @@ import ( "container/heap" "encoding/binary" "fmt" + "os" + "path/filepath" "reflect" "sort" "sync" "sync/atomic" "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/couchbase/vellum" - lev "github.com/couchbase/vellum/levenshtein" + "github.com/blevesearch/bleve/v2/document" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" + "github.com/blevesearch/vellum" + lev "github.com/blevesearch/vellum/levenshtein" + bolt "go.etcd.io/bbolt" ) // re usable, threadsafe levenshtein builders @@ -190,21 +193,62 @@ func (i *IndexSnapshot) newIndexSnapshotFieldDict(field string, func (i *IndexSnapshot) FieldDict(field string) (index.FieldDict, error) { return i.newIndexSnapshotFieldDict(field, func(i segment.TermDictionary) segment.DictionaryIterator { - return i.Iterator() + return i.AutomatonIterator(nil, nil, nil) }, false) } +// calculateExclusiveEndFromInclusiveEnd produces the next key +// when sorting using memcmp style comparisons, suitable to +// use as the end key in a traditional (inclusive, exclusive] +// start/end range +func calculateExclusiveEndFromInclusiveEnd(inclusiveEnd []byte) []byte { + rv := inclusiveEnd + if len(inclusiveEnd) > 0 { + rv = make([]byte, len(inclusiveEnd)) + copy(rv, inclusiveEnd) + if rv[len(rv)-1] < 0xff { + // last byte can be incremented by one + rv[len(rv)-1]++ + } else { + // last byte is already 0xff, so append 0 + // next key is simply one byte longer + rv = append(rv, 0x0) + } + } + return rv +} + func (i *IndexSnapshot) FieldDictRange(field string, startTerm []byte, endTerm []byte) (index.FieldDict, error) { return i.newIndexSnapshotFieldDict(field, func(i segment.TermDictionary) segment.DictionaryIterator { - return i.RangeIterator(string(startTerm), string(endTerm)) + endTermExclusive := calculateExclusiveEndFromInclusiveEnd(endTerm) + return i.AutomatonIterator(nil, startTerm, endTermExclusive) }, false) } +// calculateExclusiveEndFromPrefix produces the first key that +// does not have the same prefix as the input bytes, suitable +// to use as the end key in a traditional (inclusive, exclusive] +// start/end range +func calculateExclusiveEndFromPrefix(in []byte) []byte { + rv := make([]byte, len(in)) + copy(rv, in) + for i := len(rv) - 1; i >= 0; i-- { + rv[i] = rv[i] + 1 + if rv[i] != 0 { + return rv // didn't overflow, so stop + } + } + // all bytes were 0xff, so return nil + // as there is no end key for this prefix + return nil +} + func (i *IndexSnapshot) FieldDictPrefix(field string, termPrefix []byte) (index.FieldDict, error) { + termPrefixEnd := calculateExclusiveEndFromPrefix(termPrefix) return i.newIndexSnapshotFieldDict(field, func(i segment.TermDictionary) segment.DictionaryIterator { - return i.PrefixIterator(string(termPrefix)) + return i.AutomatonIterator(nil, termPrefix, termPrefixEnd) }, false) } @@ -213,7 +257,7 @@ func (i *IndexSnapshot) FieldDictRegexp(field string, // TODO: potential optimization where the literal prefix represents the, // entire regexp, allowing us to use PrefixIterator(prefixTerm)? - a, prefixBeg, prefixEnd, err := segment.ParseRegexp(termRegex) + a, prefixBeg, prefixEnd, err := parseRegexp(termRegex) if err != nil { return nil, err } @@ -243,7 +287,7 @@ func (i *IndexSnapshot) FieldDictFuzzy(field string, var prefixBeg, prefixEnd []byte if prefix != "" { prefixBeg = []byte(prefix) - prefixEnd = segment.IncrementBytes(prefixBeg) + prefixEnd = calculateExclusiveEndFromPrefix(prefixBeg) } return i.newIndexSnapshotFieldDict(field, func(i segment.TermDictionary) segment.DictionaryIterator { @@ -251,13 +295,6 @@ func (i *IndexSnapshot) FieldDictFuzzy(field string, }, false) } -func (i *IndexSnapshot) FieldDictOnly(field string, - onlyTerms [][]byte, includeCount bool) (index.FieldDict, error) { - return i.newIndexSnapshotFieldDict(field, func(i segment.TermDictionary) segment.DictionaryIterator { - return i.OnlyIterator(onlyTerms, includeCount) - }, false) -} - func (i *IndexSnapshot) FieldDictContains(field string) (index.FieldDictContains, error) { return i.newIndexSnapshotFieldDict(field, nil, true) } @@ -349,7 +386,7 @@ func (i *IndexSnapshot) DocCount() (uint64, error) { return rv, nil } -func (i *IndexSnapshot) Document(id string) (rv *document.Document, err error) { +func (i *IndexSnapshot) Document(id string) (rv index.Document, err error) { // FIXME could be done more efficiently directly, but reusing for simplicity tfr, err := i.TermFieldReader([]byte(id), "_id", false, false, false) if err != nil { @@ -377,7 +414,7 @@ func (i *IndexSnapshot) Document(id string) (rv *document.Document, err error) { } segmentIndex, localDocNum := i.segmentIndexAndLocalDocNumFromGlobal(docNum) - rv = document.NewDocument(id) + rvd := document.NewDocument(id) err = i.segment[segmentIndex].VisitDocument(localDocNum, func(name string, typ byte, val []byte, pos []uint64) bool { if name == "_id" { return true @@ -389,15 +426,15 @@ func (i *IndexSnapshot) Document(id string) (rv *document.Document, err error) { switch typ { case 't': - rv.AddField(document.NewTextField(name, arrayPos, value)) + rvd.AddField(document.NewTextField(name, arrayPos, value)) case 'n': - rv.AddField(document.NewNumericFieldFromBytes(name, arrayPos, value)) + rvd.AddField(document.NewNumericFieldFromBytes(name, arrayPos, value)) case 'd': - rv.AddField(document.NewDateTimeFieldFromBytes(name, arrayPos, value)) + rvd.AddField(document.NewDateTimeFieldFromBytes(name, arrayPos, value)) case 'b': - rv.AddField(document.NewBooleanFieldFromBytes(name, arrayPos, value)) + rvd.AddField(document.NewBooleanFieldFromBytes(name, arrayPos, value)) case 'g': - rv.AddField(document.NewGeoPointFieldFromBytes(name, arrayPos, value)) + rvd.AddField(document.NewGeoPointFieldFromBytes(name, arrayPos, value)) } return true @@ -406,7 +443,7 @@ func (i *IndexSnapshot) Document(id string) (rv *document.Document, err error) { return nil, err } - return rv, nil + return rvd, nil } func (i *IndexSnapshot) segmentIndexAndLocalDocNumFromGlobal(docNum uint64) (int, uint64) { @@ -563,40 +600,15 @@ func docInternalToNumber(in index.IndexInternalID) (uint64, error) { return binary.BigEndian.Uint64(in), nil } -func (i *IndexSnapshot) DocumentVisitFieldTerms(id index.IndexInternalID, - fields []string, visitor index.DocumentFieldTermVisitor) error { - _, err := i.documentVisitFieldTerms(id, fields, visitor, nil) - return err -} - -func (i *IndexSnapshot) documentVisitFieldTerms(id index.IndexInternalID, - fields []string, visitor index.DocumentFieldTermVisitor, - dvs segment.DocVisitState) (segment.DocVisitState, error) { - docNum, err := docInternalToNumber(id) - if err != nil { - return nil, err - } - - segmentIndex, localDocNum := i.segmentIndexAndLocalDocNumFromGlobal(docNum) - if segmentIndex >= len(i.segment) { - return nil, nil - } - - _, dvs, err = i.documentVisitFieldTermsOnSegment( - segmentIndex, localDocNum, fields, nil, visitor, dvs) - - return dvs, err -} - func (i *IndexSnapshot) documentVisitFieldTermsOnSegment( segmentIndex int, localDocNum uint64, fields []string, cFields []string, - visitor index.DocumentFieldTermVisitor, dvs segment.DocVisitState) ( + visitor index.DocValueVisitor, dvs segment.DocVisitState) ( cFieldsOut []string, dvsOut segment.DocVisitState, err error) { ss := i.segment[segmentIndex] var vFields []string // fields that are visitable via the segment - ssv, ssvOk := ss.segment.(segment.DocumentFieldTermVisitable) + ssv, ssvOk := ss.segment.(segment.DocValueVisitable) if ssvOk && ssv != nil { vFields, err = ssv.VisitableDocValueFields() if err != nil { @@ -627,7 +639,7 @@ func (i *IndexSnapshot) documentVisitFieldTermsOnSegment( } if ssvOk && ssv != nil && len(vFields) > 0 { - dvs, err = ssv.VisitDocumentFieldTerms(localDocNum, fields, visitor, dvs) + dvs, err = ssv.VisitDocValues(localDocNum, fields, visitor, dvs) if err != nil { return nil, nil, err } @@ -662,7 +674,7 @@ type DocValueReader struct { } func (dvr *DocValueReader) VisitDocValues(id index.IndexInternalID, - visitor index.DocumentFieldTermVisitor) (err error) { + visitor index.DocValueVisitor) (err error) { docNum, err := docInternalToNumber(id) if err != nil { return err @@ -753,3 +765,46 @@ OUTER: } return rv } + +func (i *IndexSnapshot) CopyTo(d index.Directory) error { + // get the root bolt file. + w, err := d.GetWriter(filepath.Join("store", "root.bolt")) + if err != nil || w == nil { + return fmt.Errorf("failed to create the root.bolt file, err: %v", err) + } + rootFile, ok := w.(*os.File) + if !ok { + return fmt.Errorf("invalid root.bolt file found") + } + + copyBolt, err := bolt.Open(rootFile.Name(), 0600, nil) + if err != nil { + return err + } + defer func() { + w.Close() + if cerr := copyBolt.Close(); cerr != nil && err == nil { + err = cerr + } + }() + + // start a write transaction + tx, err := copyBolt.Begin(true) + if err != nil { + return err + } + + _, _, err = prepareBoltSnapshot(i, tx, "", i.parent.segPlugin, d) + if err != nil { + _ = tx.Rollback() + return fmt.Errorf("error backing up index snapshot: %v", err) + } + + // commit bolt data + err = tx.Commit() + if err != nil { + return fmt.Errorf("error commit tx to backup root bolt: %v", err) + } + + return copyBolt.Sync() +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index_dict.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index_dict.go similarity index 96% rename from vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index_dict.go rename to vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index_dict.go index 47486c2554..0a2bd232e1 100644 --- a/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index_dict.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index_dict.go @@ -17,8 +17,8 @@ package scorch import ( "container/heap" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" ) type segmentDictCursor struct { diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index_doc.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index_doc.go similarity index 95% rename from vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index_doc.go rename to vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index_doc.go index 27da208655..fe174e7e3e 100644 --- a/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index_doc.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index_doc.go @@ -19,8 +19,8 @@ import ( "reflect" "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeIndexSnapshotDocIDReader int diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index_tfr.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index_tfr.go similarity index 97% rename from vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index_tfr.go rename to vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index_tfr.go index 239f68fbe7..e983e3dd26 100644 --- a/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index_tfr.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index_tfr.go @@ -20,9 +20,9 @@ import ( "reflect" "sync/atomic" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" ) var reflectStaticSizeIndexSnapshotTermFieldReader int diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_segment.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_segment.go similarity index 94% rename from vendor/github.com/blevesearch/bleve/index/scorch/snapshot_segment.go rename to vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_segment.go index 96742b4f94..e017eb2a92 100644 --- a/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_segment.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_segment.go @@ -20,9 +20,9 @@ import ( "sync/atomic" "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" ) var TermSeparator byte = 0xff @@ -62,8 +62,8 @@ func (s *SegmentSnapshot) Close() error { return s.segment.Close() } -func (s *SegmentSnapshot) VisitDocument(num uint64, visitor segment.DocumentFieldValueVisitor) error { - return s.segment.VisitDocument(num, visitor) +func (s *SegmentSnapshot) VisitDocument(num uint64, visitor segment.StoredFieldValueVisitor) error { + return s.segment.VisitStoredFields(num, visitor) } func (s *SegmentSnapshot) DocID(num uint64) ([]byte, error) { @@ -147,7 +147,7 @@ func (cfd *cachedFieldDocs) prepareField(field string, ss *SegmentSnapshot) { var postings segment.PostingsList var postingsItr segment.PostingsIterator - dictItr := dict.Iterator() + dictItr := dict.AutomatonIterator(nil, nil, nil) next, err := dictItr.Next() for err == nil && next != nil { var err1 error @@ -253,7 +253,7 @@ func (c *cachedDocs) updateSizeLOCKED() { } func (c *cachedDocs) visitDoc(localDocNum uint64, - fields []string, visitor index.DocumentFieldTermVisitor) { + fields []string, visitor index.DocValueVisitor) { c.m.Lock() for _, field := range fields { diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/stats.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/stats.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/index/scorch/stats.go rename to vendor/github.com/blevesearch/bleve/v2/index/scorch/stats.go diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/segment/unadorned.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/unadorned.go similarity index 64% rename from vendor/github.com/blevesearch/bleve/index/scorch/segment/unadorned.go rename to vendor/github.com/blevesearch/bleve/v2/index/scorch/unadorned.go index db06562df1..855b813613 100644 --- a/vendor/github.com/blevesearch/bleve/index/scorch/segment/unadorned.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/unadorned.go @@ -12,10 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -package segment +package scorch import ( "github.com/RoaringBitmap/roaring" + segment "github.com/blevesearch/scorch_segment_api/v2" "math" "reflect" ) @@ -25,28 +26,28 @@ var reflectStaticSizeUnadornedPostingsIterator1Hit int var reflectStaticSizeUnadornedPosting int func init() { - var pib UnadornedPostingsIteratorBitmap + var pib unadornedPostingsIteratorBitmap reflectStaticSizeUnadornedPostingsIteratorBitmap = int(reflect.TypeOf(pib).Size()) - var pi1h UnadornedPostingsIterator1Hit + var pi1h unadornedPostingsIterator1Hit reflectStaticSizeUnadornedPostingsIterator1Hit = int(reflect.TypeOf(pi1h).Size()) var up UnadornedPosting reflectStaticSizeUnadornedPosting = int(reflect.TypeOf(up).Size()) } -type UnadornedPostingsIteratorBitmap struct { +type unadornedPostingsIteratorBitmap struct { actual roaring.IntPeekable actualBM *roaring.Bitmap } -func (i *UnadornedPostingsIteratorBitmap) Next() (Posting, error) { +func (i *unadornedPostingsIteratorBitmap) Next() (segment.Posting, error) { return i.nextAtOrAfter(0) } -func (i *UnadornedPostingsIteratorBitmap) Advance(docNum uint64) (Posting, error) { +func (i *unadornedPostingsIteratorBitmap) Advance(docNum uint64) (segment.Posting, error) { return i.nextAtOrAfter(docNum) } -func (i *UnadornedPostingsIteratorBitmap) nextAtOrAfter(atOrAfter uint64) (Posting, error) { +func (i *unadornedPostingsIteratorBitmap) nextAtOrAfter(atOrAfter uint64) (segment.Posting, error) { docNum, exists := i.nextDocNumAtOrAfter(atOrAfter) if !exists { return nil, nil @@ -54,7 +55,7 @@ func (i *UnadornedPostingsIteratorBitmap) nextAtOrAfter(atOrAfter uint64) (Posti return UnadornedPosting(docNum), nil } -func (i *UnadornedPostingsIteratorBitmap) nextDocNumAtOrAfter(atOrAfter uint64) (uint64, bool) { +func (i *unadornedPostingsIteratorBitmap) nextDocNumAtOrAfter(atOrAfter uint64) (uint64, bool) { if i.actual == nil || !i.actual.HasNext() { return 0, false } @@ -67,25 +68,25 @@ func (i *UnadornedPostingsIteratorBitmap) nextDocNumAtOrAfter(atOrAfter uint64) return uint64(i.actual.Next()), true } -func (i *UnadornedPostingsIteratorBitmap) Size() int { +func (i *unadornedPostingsIteratorBitmap) Size() int { return reflectStaticSizeUnadornedPostingsIteratorBitmap } -func (i *UnadornedPostingsIteratorBitmap) ActualBitmap() *roaring.Bitmap { +func (i *unadornedPostingsIteratorBitmap) ActualBitmap() *roaring.Bitmap { return i.actualBM } -func (i *UnadornedPostingsIteratorBitmap) DocNum1Hit() (uint64, bool) { +func (i *unadornedPostingsIteratorBitmap) DocNum1Hit() (uint64, bool) { return 0, false } -func (i *UnadornedPostingsIteratorBitmap) ReplaceActual(actual *roaring.Bitmap) { +func (i *unadornedPostingsIteratorBitmap) ReplaceActual(actual *roaring.Bitmap) { i.actualBM = actual i.actual = actual.Iterator() } -func NewUnadornedPostingsIteratorFromBitmap(bm *roaring.Bitmap) PostingsIterator { - return &UnadornedPostingsIteratorBitmap{ +func newUnadornedPostingsIteratorFromBitmap(bm *roaring.Bitmap) segment.PostingsIterator { + return &unadornedPostingsIteratorBitmap{ actualBM: bm, actual: bm.Iterator(), } @@ -93,19 +94,19 @@ func NewUnadornedPostingsIteratorFromBitmap(bm *roaring.Bitmap) PostingsIterator const docNum1HitFinished = math.MaxUint64 -type UnadornedPostingsIterator1Hit struct { +type unadornedPostingsIterator1Hit struct { docNum uint64 } -func (i *UnadornedPostingsIterator1Hit) Next() (Posting, error) { +func (i *unadornedPostingsIterator1Hit) Next() (segment.Posting, error) { return i.nextAtOrAfter(0) } -func (i *UnadornedPostingsIterator1Hit) Advance(docNum uint64) (Posting, error) { +func (i *unadornedPostingsIterator1Hit) Advance(docNum uint64) (segment.Posting, error) { return i.nextAtOrAfter(docNum) } -func (i *UnadornedPostingsIterator1Hit) nextAtOrAfter(atOrAfter uint64) (Posting, error) { +func (i *unadornedPostingsIterator1Hit) nextAtOrAfter(atOrAfter uint64) (segment.Posting, error) { docNum, exists := i.nextDocNumAtOrAfter(atOrAfter) if !exists { return nil, nil @@ -113,7 +114,7 @@ func (i *UnadornedPostingsIterator1Hit) nextAtOrAfter(atOrAfter uint64) (Posting return UnadornedPosting(docNum), nil } -func (i *UnadornedPostingsIterator1Hit) nextDocNumAtOrAfter(atOrAfter uint64) (uint64, bool) { +func (i *unadornedPostingsIterator1Hit) nextDocNumAtOrAfter(atOrAfter uint64) (uint64, bool) { if i.docNum == docNum1HitFinished { return 0, false } @@ -127,12 +128,12 @@ func (i *UnadornedPostingsIterator1Hit) nextDocNumAtOrAfter(atOrAfter uint64) (u return docNum, true } -func (i *UnadornedPostingsIterator1Hit) Size() int { +func (i *unadornedPostingsIterator1Hit) Size() int { return reflectStaticSizeUnadornedPostingsIterator1Hit } -func NewUnadornedPostingsIteratorFrom1Hit(docNum1Hit uint64) PostingsIterator { - return &UnadornedPostingsIterator1Hit{ +func newUnadornedPostingsIteratorFrom1Hit(docNum1Hit uint64) segment.PostingsIterator { + return &unadornedPostingsIterator1Hit{ docNum1Hit, } } @@ -151,7 +152,7 @@ func (p UnadornedPosting) Norm() float64 { return 0 } -func (p UnadornedPosting) Locations() []Location { +func (p UnadornedPosting) Locations() []segment.Location { return nil } diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/analysis.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/analysis.go similarity index 71% rename from vendor/github.com/blevesearch/bleve/index/upsidedown/analysis.go rename to vendor/github.com/blevesearch/bleve/v2/index/upsidedown/analysis.go index d1b1fd5965..1ebd1918ee 100644 --- a/vendor/github.com/blevesearch/bleve/index/upsidedown/analysis.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/analysis.go @@ -15,29 +15,46 @@ package upsidedown import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index" + index "github.com/blevesearch/bleve_index_api" ) -func (udc *UpsideDownCouch) Analyze(d *document.Document) *index.AnalysisResult { - rv := &index.AnalysisResult{ - DocID: d.ID, - Rows: make([]index.IndexRow, 0, 100), +type IndexRow interface { + KeySize() int + KeyTo([]byte) (int, error) + Key() []byte + + ValueSize() int + ValueTo([]byte) (int, error) + Value() []byte +} + +type AnalysisResult struct { + DocID string + Rows []IndexRow +} + +func (udc *UpsideDownCouch) Analyze(d index.Document) *AnalysisResult { + return udc.analyze(d) +} + +func (udc *UpsideDownCouch) analyze(d index.Document) *AnalysisResult { + rv := &AnalysisResult{ + DocID: d.ID(), + Rows: make([]IndexRow, 0, 100), } - docIDBytes := []byte(d.ID) + docIDBytes := []byte(d.ID()) // track our back index entries backIndexStoredEntries := make([]*BackIndexStoreEntry, 0) // information we collate as we merge fields with same name - fieldTermFreqs := make(map[uint16]analysis.TokenFrequencies) + fieldTermFreqs := make(map[uint16]index.TokenFrequencies) fieldLengths := make(map[uint16]int) fieldIncludeTermVectors := make(map[uint16]bool) fieldNames := make(map[uint16]string) - analyzeField := func(field document.Field, storable bool) { + analyzeField := func(field index.Field, storable bool) { fieldIndex, newFieldRow := udc.fieldIndexOrNewRow(field.Name()) if newFieldRow != nil { rv.Rows = append(rv.Rows, newFieldRow) @@ -45,7 +62,9 @@ func (udc *UpsideDownCouch) Analyze(d *document.Document) *index.AnalysisResult fieldNames[fieldIndex] = field.Name() if field.Options().IsIndexed() { - fieldLength, tokenFreqs := field.Analyze() + field.Analyze() + fieldLength := field.AnalyzedLength() + tokenFreqs := field.AnalyzedTokenFrequencies() existingFreqs := fieldTermFreqs[fieldIndex] if existingFreqs == nil { fieldTermFreqs[fieldIndex] = tokenFreqs @@ -66,21 +85,21 @@ func (udc *UpsideDownCouch) Analyze(d *document.Document) *index.AnalysisResult // place information about indexed fields into map // this collates information across fields with // same names (arrays) - for _, field := range d.Fields { + d.VisitFields(func(field index.Field) { analyzeField(field, true) - } + }) - if len(d.CompositeFields) > 0 { + if d.HasComposite() { for fieldIndex, tokenFreqs := range fieldTermFreqs { // see if any of the composite fields need this - for _, compositeField := range d.CompositeFields { - compositeField.Compose(fieldNames[fieldIndex], fieldLengths[fieldIndex], tokenFreqs) - } + d.VisitComposite(func(field index.CompositeField) { + field.Compose(fieldNames[fieldIndex], fieldLengths[fieldIndex], tokenFreqs) + }) } - for _, compositeField := range d.CompositeFields { - analyzeField(compositeField, false) - } + d.VisitComposite(func(field index.CompositeField) { + analyzeField(field, false) + }) } rowsCapNeeded := len(rv.Rows) + 1 @@ -88,7 +107,7 @@ func (udc *UpsideDownCouch) Analyze(d *document.Document) *index.AnalysisResult rowsCapNeeded += len(tokenFreqs) } - rv.Rows = append(make([]index.IndexRow, 0, rowsCapNeeded), rv.Rows...) + rv.Rows = append(make([]IndexRow, 0, rowsCapNeeded), rv.Rows...) backIndexTermsEntries := make([]*BackIndexTermsEntry, 0, len(fieldTermFreqs)) diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/benchmark_all.sh b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/benchmark_all.sh similarity index 100% rename from vendor/github.com/blevesearch/bleve/index/upsidedown/benchmark_all.sh rename to vendor/github.com/blevesearch/bleve/v2/index/upsidedown/benchmark_all.sh diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/dump.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/dump.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/index/upsidedown/dump.go rename to vendor/github.com/blevesearch/bleve/v2/index/upsidedown/dump.go index cb045d2493..64ebb1b265 100644 --- a/vendor/github.com/blevesearch/bleve/index/upsidedown/dump.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/dump.go @@ -18,7 +18,7 @@ import ( "bytes" "sort" - "github.com/blevesearch/bleve/index/store" + "github.com/blevesearch/upsidedown_store_api" ) // the functions in this file are only intended to be used by diff --git a/vendor/github.com/blevesearch/bleve/index/field_cache.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/field_cache.go similarity index 99% rename from vendor/github.com/blevesearch/bleve/index/field_cache.go rename to vendor/github.com/blevesearch/bleve/v2/index/upsidedown/field_cache.go index 9354081f1d..1f68b71dd6 100644 --- a/vendor/github.com/blevesearch/bleve/index/field_cache.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/field_cache.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package index +package upsidedown import ( "sync" diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/field_dict.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/field_dict.go similarity index 95% rename from vendor/github.com/blevesearch/bleve/index/upsidedown/field_dict.go rename to vendor/github.com/blevesearch/bleve/v2/index/upsidedown/field_dict.go index 20d4eb34f2..c4be577403 100644 --- a/vendor/github.com/blevesearch/bleve/index/upsidedown/field_dict.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/field_dict.go @@ -17,8 +17,8 @@ package upsidedown import ( "fmt" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/store" + index "github.com/blevesearch/bleve_index_api" + store "github.com/blevesearch/upsidedown_store_api" ) type UpsideDownCouchFieldDict struct { diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/index_reader.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/index_reader.go similarity index 91% rename from vendor/github.com/blevesearch/bleve/index/upsidedown/index_reader.go rename to vendor/github.com/blevesearch/bleve/v2/index/upsidedown/index_reader.go index ea7243eaa6..ff0986d57e 100644 --- a/vendor/github.com/blevesearch/bleve/index/upsidedown/index_reader.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/index_reader.go @@ -17,9 +17,9 @@ package upsidedown import ( "reflect" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/store" + "github.com/blevesearch/bleve/v2/document" + index "github.com/blevesearch/bleve_index_api" + "github.com/blevesearch/upsidedown_store_api" ) var reflectStaticSizeIndexReader int @@ -67,7 +67,7 @@ func (i *IndexReader) DocIDReaderOnly(ids []string) (index.DocIDReader, error) { return newUpsideDownCouchDocIDReaderOnly(i, ids) } -func (i *IndexReader) Document(id string) (doc *document.Document, err error) { +func (i *IndexReader) Document(id string) (doc index.Document, err error) { // first hit the back index to confirm doc exists var backIndexRow *BackIndexRow backIndexRow, err = backIndexRowForDoc(i.kvreader, []byte(id)) @@ -77,7 +77,7 @@ func (i *IndexReader) Document(id string) (doc *document.Document, err error) { if backIndexRow == nil { return } - doc = document.NewDocument(id) + rvd := document.NewDocument(id) storedRow := NewStoredRow([]byte(id), 0, []uint64{}, 'x', nil) storedRowScanPrefix := storedRow.ScanPrefixForDoc() it := i.kvreader.PrefixIterator(storedRowScanPrefix) @@ -93,24 +93,23 @@ func (i *IndexReader) Document(id string) (doc *document.Document, err error) { var row *StoredRow row, err = NewStoredRowKV(key, safeVal) if err != nil { - doc = nil - return + return nil, err } if row != nil { fieldName := i.index.fieldCache.FieldIndexed(row.field) field := decodeFieldType(row.typ, fieldName, row.arrayPositions, row.value) if field != nil { - doc.AddField(field) + rvd.AddField(field) } } it.Next() key, val, valid = it.Current() } - return + return rvd, nil } -func (i *IndexReader) DocumentVisitFieldTerms(id index.IndexInternalID, fields []string, visitor index.DocumentFieldTermVisitor) error { +func (i *IndexReader) documentVisitFieldTerms(id index.IndexInternalID, fields []string, visitor index.DocValueVisitor) error { fieldsMap := make(map[uint16]string, len(fields)) for _, f := range fields { id, ok := i.index.fieldCache.FieldNamed(f, false) @@ -221,6 +220,6 @@ type DocValueReader struct { } func (dvr *DocValueReader) VisitDocValues(id index.IndexInternalID, - visitor index.DocumentFieldTermVisitor) error { - return dvr.i.DocumentVisitFieldTerms(id, dvr.fields, visitor) + visitor index.DocValueVisitor) error { + return dvr.i.documentVisitFieldTerms(id, dvr.fields, visitor) } diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/reader.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/reader.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/index/upsidedown/reader.go rename to vendor/github.com/blevesearch/bleve/v2/index/upsidedown/reader.go index bc0fef1199..68b15318e1 100644 --- a/vendor/github.com/blevesearch/bleve/index/upsidedown/reader.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/reader.go @@ -20,9 +20,9 @@ import ( "sort" "sync/atomic" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" + "github.com/blevesearch/upsidedown_store_api" ) var reflectStaticSizeUpsideDownCouchTermFieldReader int diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/row.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/row.go similarity index 99% rename from vendor/github.com/blevesearch/bleve/index/upsidedown/row.go rename to vendor/github.com/blevesearch/bleve/v2/index/upsidedown/row.go index 531e0a0d33..901a4bcb06 100644 --- a/vendor/github.com/blevesearch/bleve/index/upsidedown/row.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/row.go @@ -22,7 +22,7 @@ import ( "math" "reflect" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/size" "github.com/golang/protobuf/proto" ) diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/row_merge.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/row_merge.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/index/upsidedown/row_merge.go rename to vendor/github.com/blevesearch/bleve/v2/index/upsidedown/row_merge.go diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/stats.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/stats.go similarity index 97% rename from vendor/github.com/blevesearch/bleve/index/upsidedown/stats.go rename to vendor/github.com/blevesearch/bleve/v2/index/upsidedown/stats.go index a148ab7048..c1b4ddc018 100644 --- a/vendor/github.com/blevesearch/bleve/index/upsidedown/stats.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/stats.go @@ -18,7 +18,7 @@ import ( "encoding/json" "sync/atomic" - "github.com/blevesearch/bleve/index/store" + "github.com/blevesearch/upsidedown_store_api" ) type indexStat struct { diff --git a/vendor/github.com/blevesearch/bleve/index/store/boltdb/iterator.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb/iterator.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/index/store/boltdb/iterator.go rename to vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb/iterator.go diff --git a/vendor/github.com/blevesearch/bleve/index/store/boltdb/reader.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb/reader.go similarity index 96% rename from vendor/github.com/blevesearch/bleve/index/store/boltdb/reader.go rename to vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb/reader.go index 7977ebbe56..79513f600c 100644 --- a/vendor/github.com/blevesearch/bleve/index/store/boltdb/reader.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb/reader.go @@ -15,7 +15,7 @@ package boltdb import ( - "github.com/blevesearch/bleve/index/store" + store "github.com/blevesearch/upsidedown_store_api" bolt "go.etcd.io/bbolt" ) diff --git a/vendor/github.com/blevesearch/bleve/index/store/boltdb/stats.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb/stats.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/index/store/boltdb/stats.go rename to vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb/stats.go diff --git a/vendor/github.com/blevesearch/bleve/index/store/boltdb/store.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb/store.go similarity index 97% rename from vendor/github.com/blevesearch/bleve/index/store/boltdb/store.go rename to vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb/store.go index 3c749693c0..bc99275e17 100644 --- a/vendor/github.com/blevesearch/bleve/index/store/boltdb/store.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb/store.go @@ -28,8 +28,8 @@ import ( "fmt" "os" - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/registry" + "github.com/blevesearch/bleve/v2/registry" + store "github.com/blevesearch/upsidedown_store_api" bolt "go.etcd.io/bbolt" ) diff --git a/vendor/github.com/blevesearch/bleve/index/store/boltdb/writer.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb/writer.go similarity index 97% rename from vendor/github.com/blevesearch/bleve/index/store/boltdb/writer.go rename to vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb/writer.go index f09357457b..c25583cab7 100644 --- a/vendor/github.com/blevesearch/bleve/index/store/boltdb/writer.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb/writer.go @@ -17,7 +17,7 @@ package boltdb import ( "fmt" - "github.com/blevesearch/bleve/index/store" + store "github.com/blevesearch/upsidedown_store_api" ) type Writer struct { diff --git a/vendor/github.com/blevesearch/bleve/index/store/gtreap/iterator.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap/iterator.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/index/store/gtreap/iterator.go rename to vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap/iterator.go diff --git a/vendor/github.com/blevesearch/bleve/index/store/gtreap/reader.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap/reader.go similarity index 97% rename from vendor/github.com/blevesearch/bleve/index/store/gtreap/reader.go rename to vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap/reader.go index 98254d347b..34df81302b 100644 --- a/vendor/github.com/blevesearch/bleve/index/store/gtreap/reader.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap/reader.go @@ -18,7 +18,7 @@ package gtreap import ( - "github.com/blevesearch/bleve/index/store" + "github.com/blevesearch/upsidedown_store_api" "github.com/steveyen/gtreap" ) diff --git a/vendor/github.com/blevesearch/bleve/index/store/gtreap/store.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap/store.go similarity index 94% rename from vendor/github.com/blevesearch/bleve/index/store/gtreap/store.go rename to vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap/store.go index 3e6c5fec1a..8d1f5887d6 100644 --- a/vendor/github.com/blevesearch/bleve/index/store/gtreap/store.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap/store.go @@ -24,8 +24,8 @@ import ( "os" "sync" - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/registry" + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/upsidedown_store_api" "github.com/steveyen/gtreap" ) diff --git a/vendor/github.com/blevesearch/bleve/index/store/gtreap/writer.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap/writer.go similarity index 97% rename from vendor/github.com/blevesearch/bleve/index/store/gtreap/writer.go rename to vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap/writer.go index 777aab406d..80aa15b193 100644 --- a/vendor/github.com/blevesearch/bleve/index/store/gtreap/writer.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap/writer.go @@ -21,7 +21,7 @@ import ( "fmt" "math/rand" - "github.com/blevesearch/bleve/index/store" + "github.com/blevesearch/upsidedown_store_api" ) type Writer struct { diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/upsidedown.go similarity index 92% rename from vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.go rename to vendor/github.com/blevesearch/bleve/v2/index/upsidedown/upsidedown.go index 8e915c6ad5..d67ee03fb0 100644 --- a/vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/upsidedown.go @@ -25,11 +25,10 @@ import ( "sync/atomic" "time" - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/registry" + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/registry" + index "github.com/blevesearch/bleve_index_api" + store "github.com/blevesearch/upsidedown_store_api" "github.com/golang/protobuf/proto" ) @@ -49,13 +48,15 @@ const Version uint8 = 7 var IncompatibleVersion = fmt.Errorf("incompatible version, %d is supported", Version) +var ErrorUnknownStorageType = fmt.Errorf("unknown storage type") + type UpsideDownCouch struct { version uint8 path string storeName string storeConfig map[string]interface{} store store.KVStore - fieldCache *index.FieldCache + fieldCache *FieldCache analysisQueue *index.AnalysisQueue stats *indexStat @@ -68,14 +69,14 @@ type UpsideDownCouch struct { type docBackIndexRow struct { docID string - doc *document.Document // If deletion, doc will be nil. + doc index.Document // If deletion, doc will be nil. backIndexRow *BackIndexRow } func NewUpsideDownCouch(storeName string, storeConfig map[string]interface{}, analysisQueue *index.AnalysisQueue) (index.Index, error) { rv := &UpsideDownCouch{ version: Version, - fieldCache: index.NewFieldCache(), + fieldCache: NewFieldCache(), storeName: storeName, storeConfig: storeConfig, analysisQueue: analysisQueue, @@ -300,7 +301,7 @@ func (udc *UpsideDownCouch) Open() (err error) { // open the kv store storeConstructor := registry.KVStoreConstructorByName(udc.storeName) if storeConstructor == nil { - err = index.ErrorUnknownStorageType + err = ErrorUnknownStorageType return } @@ -412,14 +413,16 @@ func (udc *UpsideDownCouch) Close() error { return udc.store.Close() } -func (udc *UpsideDownCouch) Update(doc *document.Document) (err error) { +func (udc *UpsideDownCouch) Update(doc index.Document) (err error) { // do analysis before acquiring write lock analysisStart := time.Now() - resultChan := make(chan *index.AnalysisResult) - aw := index.NewAnalysisWork(udc, doc, resultChan) + resultChan := make(chan *AnalysisResult) // put the work on the queue - udc.analysisQueue.Queue(aw) + udc.analysisQueue.Queue(func() { + ar := udc.analyze(doc) + resultChan <- ar + }) // wait for the result result := <-resultChan @@ -439,7 +442,7 @@ func (udc *UpsideDownCouch) Update(doc *document.Document) (err error) { // first we lookup the backindex row for the doc id if it exists // lookup the back index row var backIndexRow *BackIndexRow - backIndexRow, err = backIndexRowForDoc(kvreader, index.IndexInternalID(doc.ID)) + backIndexRow, err = backIndexRowForDoc(kvreader, index.IndexInternalID(doc.ID())) if err != nil { _ = kvreader.Close() atomic.AddUint64(&udc.stats.errors, 1) @@ -454,8 +457,8 @@ func (udc *UpsideDownCouch) Update(doc *document.Document) (err error) { return udc.UpdateWithAnalysis(doc, result, backIndexRow) } -func (udc *UpsideDownCouch) UpdateWithAnalysis(doc *document.Document, - result *index.AnalysisResult, backIndexRow *BackIndexRow) (err error) { +func (udc *UpsideDownCouch) UpdateWithAnalysis(doc index.Document, + result *AnalysisResult, backIndexRow *BackIndexRow) (err error) { // start a writer for this update indexStart := time.Now() var kvwriter store.KVWriter @@ -501,7 +504,7 @@ func (udc *UpsideDownCouch) UpdateWithAnalysis(doc *document.Document, return } -func (udc *UpsideDownCouch) mergeOldAndNew(backIndexRow *BackIndexRow, rows []index.IndexRow) (addRows []UpsideDownCouchRow, updateRows []UpsideDownCouchRow, deleteRows []UpsideDownCouchRow) { +func (udc *UpsideDownCouch) mergeOldAndNew(backIndexRow *BackIndexRow, rows []IndexRow) (addRows []UpsideDownCouchRow, updateRows []UpsideDownCouchRow, deleteRows []UpsideDownCouchRow) { addRows = make([]UpsideDownCouchRow, 0, len(rows)) if backIndexRow == nil { @@ -587,8 +590,8 @@ func (udc *UpsideDownCouch) mergeOldAndNew(backIndexRow *BackIndexRow, rows []in return addRows, updateRows, deleteRows } -func (udc *UpsideDownCouch) storeField(docID []byte, field document.Field, fieldIndex uint16, rows []index.IndexRow, backIndexStoredEntries []*BackIndexStoreEntry) ([]index.IndexRow, []*BackIndexStoreEntry) { - fieldType := encodeFieldType(field) +func (udc *UpsideDownCouch) storeField(docID []byte, field index.Field, fieldIndex uint16, rows []IndexRow, backIndexStoredEntries []*BackIndexStoreEntry) ([]IndexRow, []*BackIndexStoreEntry) { + fieldType := field.EncodedFieldType() storedRow := NewStoredRow(docID, fieldIndex, field.ArrayPositions(), fieldType, field.Value()) // record the back index entry @@ -597,26 +600,7 @@ func (udc *UpsideDownCouch) storeField(docID []byte, field document.Field, field return append(rows, storedRow), append(backIndexStoredEntries, &backIndexStoredEntry) } -func encodeFieldType(f document.Field) byte { - fieldType := byte('x') - switch f.(type) { - case *document.TextField: - fieldType = 't' - case *document.NumericField: - fieldType = 'n' - case *document.DateTimeField: - fieldType = 'd' - case *document.BooleanField: - fieldType = 'b' - case *document.GeoPointField: - fieldType = 'g' - case *document.CompositeField: - fieldType = 'c' - } - return fieldType -} - -func (udc *UpsideDownCouch) indexField(docID []byte, includeTermVectors bool, fieldIndex uint16, fieldLength int, tokenFreqs analysis.TokenFrequencies, rows []index.IndexRow, backIndexTermsEntries []*BackIndexTermsEntry) ([]index.IndexRow, []*BackIndexTermsEntry) { +func (udc *UpsideDownCouch) indexField(docID []byte, includeTermVectors bool, fieldIndex uint16, fieldLength int, tokenFreqs index.TokenFrequencies, rows []IndexRow, backIndexTermsEntries []*BackIndexTermsEntry) ([]IndexRow, []*BackIndexTermsEntry) { fieldNorm := float32(1.0 / math.Sqrt(float64(fieldLength))) termFreqRows := make([]TermFrequencyRow, len(tokenFreqs)) @@ -747,11 +731,11 @@ func decodeFieldType(typ byte, name string, pos []uint64, value []byte) document return nil } -func frequencyFromTokenFreq(tf *analysis.TokenFreq) int { +func frequencyFromTokenFreq(tf *index.TokenFreq) int { return tf.Frequency() } -func (udc *UpsideDownCouch) termVectorsFromTokenFreq(field uint16, tf *analysis.TokenFreq, rows []index.IndexRow) ([]*TermVector, []index.IndexRow) { +func (udc *UpsideDownCouch) termVectorsFromTokenFreq(field uint16, tf *index.TokenFreq, rows []IndexRow) ([]*TermVector, []IndexRow) { a := make([]TermVector, len(tf.Locations)) rv := make([]*TermVector, len(tf.Locations)) @@ -807,7 +791,7 @@ func (udc *UpsideDownCouch) Batch(batch *index.Batch) (err error) { } analysisStart := time.Now() - resultChan := make(chan *index.AnalysisResult, len(batch.IndexOps)) + resultChan := make(chan *AnalysisResult, len(batch.IndexOps)) var numUpdates uint64 var numPlainTextBytes uint64 @@ -823,9 +807,11 @@ func (udc *UpsideDownCouch) Batch(batch *index.Batch) (err error) { for k := range batch.IndexOps { doc := batch.IndexOps[k] if doc != nil { - aw := index.NewAnalysisWork(udc, doc, resultChan) // put the work on the queue - udc.analysisQueue.Queue(aw) + udc.analysisQueue.Queue(func() { + ar := udc.analyze(doc) + resultChan <- ar + }) } } }() @@ -866,7 +852,7 @@ func (udc *UpsideDownCouch) Batch(batch *index.Batch) (err error) { }() // wait for analysis result - newRowsMap := make(map[string][]index.IndexRow) + newRowsMap := make(map[string][]IndexRow) var itemsDeQueued uint64 for itemsDeQueued < numUpdates { result := <-resultChan diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.pb.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/upsidedown.pb.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.pb.go rename to vendor/github.com/blevesearch/bleve/v2/index/upsidedown/upsidedown.pb.go diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.proto b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/upsidedown.proto similarity index 100% rename from vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.proto rename to vendor/github.com/blevesearch/bleve/v2/index/upsidedown/upsidedown.proto diff --git a/vendor/github.com/blevesearch/bleve/index_alias.go b/vendor/github.com/blevesearch/bleve/v2/index_alias.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/index_alias.go rename to vendor/github.com/blevesearch/bleve/v2/index_alias.go diff --git a/vendor/github.com/blevesearch/bleve/index_alias_impl.go b/vendor/github.com/blevesearch/bleve/v2/index_alias_impl.go similarity index 96% rename from vendor/github.com/blevesearch/bleve/index_alias_impl.go rename to vendor/github.com/blevesearch/bleve/v2/index_alias_impl.go index 5aa57d8ac8..5a4dc5a480 100644 --- a/vendor/github.com/blevesearch/bleve/index_alias_impl.go +++ b/vendor/github.com/blevesearch/bleve/v2/index_alias_impl.go @@ -19,11 +19,9 @@ import ( "sync" "time" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" ) type indexAliasImpl struct { @@ -110,7 +108,7 @@ func (i *indexAliasImpl) Batch(b *Batch) error { return i.indexes[0].Batch(b) } -func (i *indexAliasImpl) Document(id string) (*document.Document, error) { +func (i *indexAliasImpl) Document(id string) (index.Document, error) { i.mutex.RLock() defer i.mutex.RUnlock() @@ -369,17 +367,17 @@ func (i *indexAliasImpl) DeleteInternal(key []byte) error { return i.indexes[0].DeleteInternal(key) } -func (i *indexAliasImpl) Advanced() (index.Index, store.KVStore, error) { +func (i *indexAliasImpl) Advanced() (index.Index, error) { i.mutex.RLock() defer i.mutex.RUnlock() if !i.open { - return nil, nil, ErrorIndexClosed + return nil, ErrorIndexClosed } err := i.isAliasToSingleIndex() if err != nil { - return nil, nil, err + return nil, err } return i.indexes[0].Advanced() diff --git a/vendor/github.com/blevesearch/bleve/index_impl.go b/vendor/github.com/blevesearch/bleve/v2/index_impl.go similarity index 91% rename from vendor/github.com/blevesearch/bleve/index_impl.go rename to vendor/github.com/blevesearch/bleve/v2/index_impl.go index 0520fe43bf..8a9cfd3b42 100644 --- a/vendor/github.com/blevesearch/bleve/index_impl.go +++ b/vendor/github.com/blevesearch/bleve/v2/index_impl.go @@ -18,21 +18,22 @@ import ( "context" "encoding/json" "fmt" + "io" "os" + "path/filepath" "sync" "sync/atomic" "time" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/index/upsidedown" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/registry" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/collector" - "github.com/blevesearch/bleve/search/facet" - "github.com/blevesearch/bleve/search/highlight" + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/index/upsidedown" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/collector" + "github.com/blevesearch/bleve/v2/search/facet" + "github.com/blevesearch/bleve/v2/search/highlight" + index "github.com/blevesearch/bleve_index_api" ) type indexImpl struct { @@ -107,9 +108,6 @@ func newIndexUsing(path string, mapping mapping.IndexMapping, indexType string, } err = rv.i.Open() if err != nil { - if err == index.ErrorUnknownStorageType { - return nil, ErrorUnknownStorageType - } return nil, err } defer func(rv *indexImpl) { @@ -177,9 +175,6 @@ func openIndexUsing(path string, runtimeConfig map[string]interface{}) (rv *inde } err = rv.i.Open() if err != nil { - if err == index.ErrorUnknownStorageType { - return nil, ErrorUnknownStorageType - } return nil, err } defer func(rv *indexImpl) { @@ -228,14 +223,9 @@ func openIndexUsing(path string, runtimeConfig map[string]interface{}) (rv *inde return rv, err } -// Advanced returns implementation internals -// necessary ONLY for advanced usage. -func (i *indexImpl) Advanced() (index.Index, store.KVStore, error) { - s, err := i.i.Advanced() - if err != nil { - return nil, nil, err - } - return i.i, s, nil +// Advanced returns internal index implementation +func (i *indexImpl) Advanced() (index.Index, error) { + return i.i, nil } // Mapping returns the IndexMapping in use by this @@ -271,7 +261,7 @@ func (i *indexImpl) Index(id string, data interface{}) (err error) { // IndexAdvanced takes a document.Document object // skips the mapping and indexes it. func (i *indexImpl) IndexAdvanced(doc *document.Document) (err error) { - if doc.ID == "" { + if doc.ID() == "" { return ErrorEmptyID } @@ -323,7 +313,7 @@ func (i *indexImpl) Batch(b *Batch) error { // stored fields for a document in the index. These // stored fields are put back into a Document object // and returned. -func (i *indexImpl) Document(id string) (doc *document.Document, err error) { +func (i *indexImpl) Document(id string) (doc index.Document, err error) { i.mutex.RLock() defer i.mutex.RUnlock() @@ -617,28 +607,28 @@ func LoadAndHighlightFields(hit *search.DocumentMatch, req *SearchRequest, if len(req.Fields) > 0 { fieldsToLoad := deDuplicate(req.Fields) for _, f := range fieldsToLoad { - for _, docF := range doc.Fields { + doc.VisitFields(func(docF index.Field) { if f == "*" || docF.Name() == f { var value interface{} switch docF := docF.(type) { - case *document.TextField: - value = string(docF.Value()) - case *document.NumericField: + case index.TextField: + value = docF.Text() + case index.NumericField: num, err := docF.Number() if err == nil { value = num } - case *document.DateTimeField: + case index.DateTimeField: datetime, err := docF.DateTime() if err == nil { value = datetime.Format(time.RFC3339) } - case *document.BooleanField: + case index.BooleanField: boolean, err := docF.Boolean() if err == nil { value = boolean } - case *document.GeoPointField: + case index.GeoPointField: lon, err := docF.Lon() if err == nil { lat, err := docF.Lat() @@ -651,7 +641,7 @@ func LoadAndHighlightFields(hit *search.DocumentMatch, req *SearchRequest, hit.AddFieldValue(docF.Name(), value) } } - } + }) } } if highlighter != nil { @@ -922,3 +912,49 @@ func (m *searchHitSorter) Less(i, j int) bool { c := m.sort.Compare(m.cachedScoring, m.cachedDesc, m.hits[i], m.hits[j]) return c < 0 } + +func (i *indexImpl) CopyTo(d index.Directory) (err error) { + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return ErrorIndexClosed + } + + indexReader, err := i.i.Reader() + if err != nil { + return err + } + defer func() { + if cerr := indexReader.Close(); err == nil && cerr != nil { + err = cerr + } + }() + + irc, ok := indexReader.(IndexCopyable) + if !ok { + return fmt.Errorf("index implementation does not support copy") + } + + err = irc.CopyTo(d) + if err != nil { + return fmt.Errorf("error copying index metadata: %v", err) + } + + // copy the metadata + return i.meta.CopyTo(d) +} + +func (f FileSystemDirectory) GetWriter(filePath string) (io.WriteCloser, + error) { + dir, file := filepath.Split(filePath) + if dir != "" { + err := os.MkdirAll(filepath.Join(string(f), dir), os.ModePerm) + if err != nil { + return nil, err + } + } + + return os.OpenFile(filepath.Join(string(f), dir, file), + os.O_RDWR|os.O_CREATE, 0600) +} diff --git a/vendor/github.com/blevesearch/bleve/index_meta.go b/vendor/github.com/blevesearch/bleve/v2/index_meta.go similarity index 84% rename from vendor/github.com/blevesearch/bleve/index_meta.go rename to vendor/github.com/blevesearch/bleve/v2/index_meta.go index d814799a89..711e0c1bd6 100644 --- a/vendor/github.com/blevesearch/bleve/index_meta.go +++ b/vendor/github.com/blevesearch/bleve/v2/index_meta.go @@ -16,11 +16,13 @@ package bleve import ( "encoding/json" + "fmt" "io/ioutil" "os" "path/filepath" - "github.com/blevesearch/bleve/index/upsidedown" + "github.com/blevesearch/bleve/v2/index/upsidedown" + index "github.com/blevesearch/bleve_index_api" ) const metaFilename = "index_meta.json" @@ -92,6 +94,23 @@ func (i *indexMeta) Save(path string) (err error) { return nil } +func (i *indexMeta) CopyTo(d index.Directory) (err error) { + metaBytes, err := json.Marshal(i) + if err != nil { + return err + } + + w, err := d.GetWriter(metaFilename) + if w == nil || err != nil { + return fmt.Errorf("invalid writer for file: %s, err: %v", + metaFilename, err) + } + defer w.Close() + + _, err = w.Write(metaBytes) + return err +} + func indexMetaPath(path string) string { return filepath.Join(path, metaFilename) } diff --git a/vendor/github.com/blevesearch/bleve/index_stats.go b/vendor/github.com/blevesearch/bleve/v2/index_stats.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/index_stats.go rename to vendor/github.com/blevesearch/bleve/v2/index_stats.go diff --git a/vendor/github.com/blevesearch/bleve/mapping.go b/vendor/github.com/blevesearch/bleve/v2/mapping.go similarity index 97% rename from vendor/github.com/blevesearch/bleve/mapping.go rename to vendor/github.com/blevesearch/bleve/v2/mapping.go index 76238dc153..54753073a6 100644 --- a/vendor/github.com/blevesearch/bleve/mapping.go +++ b/vendor/github.com/blevesearch/bleve/v2/mapping.go @@ -14,7 +14,7 @@ package bleve -import "github.com/blevesearch/bleve/mapping" +import "github.com/blevesearch/bleve/v2/mapping" // NewIndexMapping creates a new IndexMapping that will use all the default indexing rules func NewIndexMapping() *mapping.IndexMappingImpl { diff --git a/vendor/github.com/blevesearch/bleve/mapping/analysis.go b/vendor/github.com/blevesearch/bleve/v2/mapping/analysis.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/mapping/analysis.go rename to vendor/github.com/blevesearch/bleve/v2/mapping/analysis.go diff --git a/vendor/github.com/blevesearch/bleve/mapping/document.go b/vendor/github.com/blevesearch/bleve/v2/mapping/document.go similarity index 99% rename from vendor/github.com/blevesearch/bleve/mapping/document.go rename to vendor/github.com/blevesearch/bleve/v2/mapping/document.go index dd42fab960..44911de77c 100644 --- a/vendor/github.com/blevesearch/bleve/mapping/document.go +++ b/vendor/github.com/blevesearch/bleve/v2/mapping/document.go @@ -21,7 +21,7 @@ import ( "reflect" "time" - "github.com/blevesearch/bleve/registry" + "github.com/blevesearch/bleve/v2/registry" ) // A DocumentMapping describes how a type of document diff --git a/vendor/github.com/blevesearch/bleve/mapping/field.go b/vendor/github.com/blevesearch/bleve/v2/mapping/field.go similarity index 91% rename from vendor/github.com/blevesearch/bleve/mapping/field.go rename to vendor/github.com/blevesearch/bleve/v2/mapping/field.go index 278faa1a92..7ad1744d4e 100644 --- a/vendor/github.com/blevesearch/bleve/mapping/field.go +++ b/vendor/github.com/blevesearch/bleve/v2/mapping/field.go @@ -19,9 +19,11 @@ import ( "fmt" "time" - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/geo" + index "github.com/blevesearch/bleve_index_api" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/geo" ) // control the default behavior for dynamic fields (those not explicitly mapped) @@ -59,6 +61,12 @@ type FieldMapping struct { // DocValues, if true makes the index uninverting possible for this field // It is useful for faceting and sorting queries. DocValues bool `json:"docvalues,omitempty"` + + // SkipFreqNorm, if true, avoids the indexing of frequency and norm values + // of the tokens for this field. This option would be useful for saving + // the processing of freq/norm details when the default score based relevancy + // isn't needed. + SkipFreqNorm bool `json:"skip_freq_norm,omitempty"` } // NewTextFieldMapping returns a default field mapping for text @@ -150,19 +158,22 @@ func NewGeoPointFieldMapping() *FieldMapping { } // Options returns the indexing options for this field. -func (fm *FieldMapping) Options() document.IndexingOptions { - var rv document.IndexingOptions +func (fm *FieldMapping) Options() index.FieldIndexingOptions { + var rv index.FieldIndexingOptions if fm.Store { - rv |= document.StoreField + rv |= index.StoreField } if fm.Index { - rv |= document.IndexField + rv |= index.IndexField } if fm.IncludeTermVectors { - rv |= document.IncludeTermVectors + rv |= index.IncludeTermVectors } if fm.DocValues { - rv |= document.DocValues + rv |= index.DocValues + } + if fm.SkipFreqNorm { + rv |= index.SkipFreqNorm } return rv } @@ -330,6 +341,11 @@ func (fm *FieldMapping) UnmarshalJSON(data []byte) error { if err != nil { return err } + case "skip_freq_norm": + err := json.Unmarshal(v, &fm.SkipFreqNorm) + if err != nil { + return err + } default: invalidKeys = append(invalidKeys, k) } diff --git a/vendor/github.com/blevesearch/bleve/mapping/index.go b/vendor/github.com/blevesearch/bleve/v2/mapping/index.go similarity index 95% rename from vendor/github.com/blevesearch/bleve/mapping/index.go rename to vendor/github.com/blevesearch/bleve/v2/mapping/index.go index 319ba949c8..c343433857 100644 --- a/vendor/github.com/blevesearch/bleve/mapping/index.go +++ b/vendor/github.com/blevesearch/bleve/v2/mapping/index.go @@ -17,12 +17,13 @@ package mapping import ( "encoding/json" "fmt" + index "github.com/blevesearch/bleve_index_api" - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/analyzer/standard" - "github.com/blevesearch/bleve/analysis/datetime/optional" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/registry" + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/analyzer/standard" + "github.com/blevesearch/bleve/v2/analysis/datetime/optional" + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/registry" ) var MappingJSONStrict = false @@ -106,10 +107,10 @@ func (im *IndexMappingImpl) AddCustomTokenFilter(name string, config map[string] // use their metadata to fill configuration entries: // // import ( -// "github.com/blevesearch/bleve/analysis/analyzer/custom" -// "github.com/blevesearch/bleve/analysis/char/html" -// "github.com/blevesearch/bleve/analysis/token/lowercase" -// "github.com/blevesearch/bleve/analysis/tokenizer/unicode" +// "github.com/blevesearch/bleve/v2/analysis/analyzer/custom" +// "github.com/blevesearch/bleve/v2/analysis/char/html" +// "github.com/blevesearch/bleve/v2/analysis/token/lowercase" +// "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" // ) // // m := bleve.NewIndexMapping() @@ -327,7 +328,7 @@ func (im *IndexMappingImpl) MapDocument(doc *document.Document, data interface{} // see if the _all field was disabled allMapping := docMapping.documentMappingForPath("_all") if allMapping == nil || allMapping.Enabled { - field := document.NewCompositeFieldWithIndexingOptions("_all", true, []string{}, walkContext.excludedFromAll, document.IndexField|document.IncludeTermVectors) + field := document.NewCompositeFieldWithIndexingOptions("_all", true, []string{}, walkContext.excludedFromAll, index.IndexField|index.IncludeTermVectors) doc.AddField(field) } } diff --git a/vendor/github.com/blevesearch/bleve/mapping/mapping.go b/vendor/github.com/blevesearch/bleve/v2/mapping/mapping.go similarity index 95% rename from vendor/github.com/blevesearch/bleve/mapping/mapping.go rename to vendor/github.com/blevesearch/bleve/v2/mapping/mapping.go index 4a47281126..bcf6749a44 100644 --- a/vendor/github.com/blevesearch/bleve/mapping/mapping.go +++ b/vendor/github.com/blevesearch/bleve/v2/mapping/mapping.go @@ -18,8 +18,8 @@ import ( "io/ioutil" "log" - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/document" + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/document" ) // A Classifier is an interface describing any object which knows how to diff --git a/vendor/github.com/blevesearch/bleve/mapping/reflect.go b/vendor/github.com/blevesearch/bleve/v2/mapping/reflect.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/mapping/reflect.go rename to vendor/github.com/blevesearch/bleve/v2/mapping/reflect.go diff --git a/vendor/github.com/blevesearch/bleve/numeric/bin.go b/vendor/github.com/blevesearch/bleve/v2/numeric/bin.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/numeric/bin.go rename to vendor/github.com/blevesearch/bleve/v2/numeric/bin.go diff --git a/vendor/github.com/blevesearch/bleve/numeric/float.go b/vendor/github.com/blevesearch/bleve/v2/numeric/float.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/numeric/float.go rename to vendor/github.com/blevesearch/bleve/v2/numeric/float.go diff --git a/vendor/github.com/blevesearch/bleve/numeric/prefix_coded.go b/vendor/github.com/blevesearch/bleve/v2/numeric/prefix_coded.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/numeric/prefix_coded.go rename to vendor/github.com/blevesearch/bleve/v2/numeric/prefix_coded.go diff --git a/vendor/github.com/blevesearch/bleve/query.go b/vendor/github.com/blevesearch/bleve/v2/query.go similarity index 99% rename from vendor/github.com/blevesearch/bleve/query.go rename to vendor/github.com/blevesearch/bleve/v2/query.go index 523db5ec05..91d2d5f421 100644 --- a/vendor/github.com/blevesearch/bleve/query.go +++ b/vendor/github.com/blevesearch/bleve/v2/query.go @@ -17,7 +17,7 @@ package bleve import ( "time" - "github.com/blevesearch/bleve/search/query" + "github.com/blevesearch/bleve/v2/search/query" ) // NewBoolFieldQuery creates a new Query for boolean fields diff --git a/vendor/github.com/blevesearch/bleve/registry/analyzer.go b/vendor/github.com/blevesearch/bleve/v2/registry/analyzer.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/registry/analyzer.go rename to vendor/github.com/blevesearch/bleve/v2/registry/analyzer.go index 340e349dc0..6c0d73b315 100644 --- a/vendor/github.com/blevesearch/bleve/registry/analyzer.go +++ b/vendor/github.com/blevesearch/bleve/v2/registry/analyzer.go @@ -17,7 +17,7 @@ package registry import ( "fmt" - "github.com/blevesearch/bleve/analysis" + "github.com/blevesearch/bleve/v2/analysis" ) func RegisterAnalyzer(name string, constructor AnalyzerConstructor) { diff --git a/vendor/github.com/blevesearch/bleve/registry/cache.go b/vendor/github.com/blevesearch/bleve/v2/registry/cache.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/registry/cache.go rename to vendor/github.com/blevesearch/bleve/v2/registry/cache.go diff --git a/vendor/github.com/blevesearch/bleve/registry/char_filter.go b/vendor/github.com/blevesearch/bleve/v2/registry/char_filter.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/registry/char_filter.go rename to vendor/github.com/blevesearch/bleve/v2/registry/char_filter.go index 4696713cc7..aa400be682 100644 --- a/vendor/github.com/blevesearch/bleve/registry/char_filter.go +++ b/vendor/github.com/blevesearch/bleve/v2/registry/char_filter.go @@ -17,7 +17,7 @@ package registry import ( "fmt" - "github.com/blevesearch/bleve/analysis" + "github.com/blevesearch/bleve/v2/analysis" ) func RegisterCharFilter(name string, constructor CharFilterConstructor) { diff --git a/vendor/github.com/blevesearch/bleve/registry/datetime_parser.go b/vendor/github.com/blevesearch/bleve/v2/registry/datetime_parser.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/registry/datetime_parser.go rename to vendor/github.com/blevesearch/bleve/v2/registry/datetime_parser.go index 2cd46e5acf..a2d8ac24ab 100644 --- a/vendor/github.com/blevesearch/bleve/registry/datetime_parser.go +++ b/vendor/github.com/blevesearch/bleve/v2/registry/datetime_parser.go @@ -17,7 +17,7 @@ package registry import ( "fmt" - "github.com/blevesearch/bleve/analysis" + "github.com/blevesearch/bleve/v2/analysis" ) func RegisterDateTimeParser(name string, constructor DateTimeParserConstructor) { diff --git a/vendor/github.com/blevesearch/bleve/registry/fragment_formatter.go b/vendor/github.com/blevesearch/bleve/v2/registry/fragment_formatter.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/registry/fragment_formatter.go rename to vendor/github.com/blevesearch/bleve/v2/registry/fragment_formatter.go index d0121d9679..6699f53bab 100644 --- a/vendor/github.com/blevesearch/bleve/registry/fragment_formatter.go +++ b/vendor/github.com/blevesearch/bleve/v2/registry/fragment_formatter.go @@ -17,7 +17,7 @@ package registry import ( "fmt" - "github.com/blevesearch/bleve/search/highlight" + "github.com/blevesearch/bleve/v2/search/highlight" ) func RegisterFragmentFormatter(name string, constructor FragmentFormatterConstructor) { diff --git a/vendor/github.com/blevesearch/bleve/registry/fragmenter.go b/vendor/github.com/blevesearch/bleve/v2/registry/fragmenter.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/registry/fragmenter.go rename to vendor/github.com/blevesearch/bleve/v2/registry/fragmenter.go index 18ab2ac070..cd1e32d28d 100644 --- a/vendor/github.com/blevesearch/bleve/registry/fragmenter.go +++ b/vendor/github.com/blevesearch/bleve/v2/registry/fragmenter.go @@ -17,7 +17,7 @@ package registry import ( "fmt" - "github.com/blevesearch/bleve/search/highlight" + "github.com/blevesearch/bleve/v2/search/highlight" ) func RegisterFragmenter(name string, constructor FragmenterConstructor) { diff --git a/vendor/github.com/blevesearch/bleve/registry/highlighter.go b/vendor/github.com/blevesearch/bleve/v2/registry/highlighter.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/registry/highlighter.go rename to vendor/github.com/blevesearch/bleve/v2/registry/highlighter.go index b84219ca11..8eb210fb3c 100644 --- a/vendor/github.com/blevesearch/bleve/registry/highlighter.go +++ b/vendor/github.com/blevesearch/bleve/v2/registry/highlighter.go @@ -17,7 +17,7 @@ package registry import ( "fmt" - "github.com/blevesearch/bleve/search/highlight" + "github.com/blevesearch/bleve/v2/search/highlight" ) func RegisterHighlighter(name string, constructor HighlighterConstructor) { diff --git a/vendor/github.com/blevesearch/bleve/registry/index_type.go b/vendor/github.com/blevesearch/bleve/v2/registry/index_type.go similarity index 96% rename from vendor/github.com/blevesearch/bleve/registry/index_type.go rename to vendor/github.com/blevesearch/bleve/v2/registry/index_type.go index 4da07c82e7..67938c4afa 100644 --- a/vendor/github.com/blevesearch/bleve/registry/index_type.go +++ b/vendor/github.com/blevesearch/bleve/v2/registry/index_type.go @@ -17,7 +17,7 @@ package registry import ( "fmt" - "github.com/blevesearch/bleve/index" + index "github.com/blevesearch/bleve_index_api" ) func RegisterIndexType(name string, constructor IndexTypeConstructor) { diff --git a/vendor/github.com/blevesearch/bleve/registry/registry.go b/vendor/github.com/blevesearch/bleve/v2/registry/registry.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/registry/registry.go rename to vendor/github.com/blevesearch/bleve/v2/registry/registry.go index a0ea69c8b8..02125e64fa 100644 --- a/vendor/github.com/blevesearch/bleve/registry/registry.go +++ b/vendor/github.com/blevesearch/bleve/v2/registry/registry.go @@ -17,8 +17,8 @@ package registry import ( "fmt" - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/search/highlight" + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/search/highlight" ) var stores = make(KVStoreRegistry, 0) diff --git a/vendor/github.com/blevesearch/bleve/registry/store.go b/vendor/github.com/blevesearch/bleve/v2/registry/store.go similarity index 97% rename from vendor/github.com/blevesearch/bleve/registry/store.go rename to vendor/github.com/blevesearch/bleve/v2/registry/store.go index 83187763d6..02ebd888cc 100644 --- a/vendor/github.com/blevesearch/bleve/registry/store.go +++ b/vendor/github.com/blevesearch/bleve/v2/registry/store.go @@ -17,7 +17,7 @@ package registry import ( "fmt" - "github.com/blevesearch/bleve/index/store" + "github.com/blevesearch/upsidedown_store_api" ) func RegisterKVStore(name string, constructor KVStoreConstructor) { diff --git a/vendor/github.com/blevesearch/bleve/registry/token_filter.go b/vendor/github.com/blevesearch/bleve/v2/registry/token_filter.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/registry/token_filter.go rename to vendor/github.com/blevesearch/bleve/v2/registry/token_filter.go index e202e15fc7..df39411ae3 100644 --- a/vendor/github.com/blevesearch/bleve/registry/token_filter.go +++ b/vendor/github.com/blevesearch/bleve/v2/registry/token_filter.go @@ -17,7 +17,7 @@ package registry import ( "fmt" - "github.com/blevesearch/bleve/analysis" + "github.com/blevesearch/bleve/v2/analysis" ) func RegisterTokenFilter(name string, constructor TokenFilterConstructor) { diff --git a/vendor/github.com/blevesearch/bleve/registry/token_maps.go b/vendor/github.com/blevesearch/bleve/v2/registry/token_maps.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/registry/token_maps.go rename to vendor/github.com/blevesearch/bleve/v2/registry/token_maps.go index 66ca08fdfd..08c9956eb1 100644 --- a/vendor/github.com/blevesearch/bleve/registry/token_maps.go +++ b/vendor/github.com/blevesearch/bleve/v2/registry/token_maps.go @@ -17,7 +17,7 @@ package registry import ( "fmt" - "github.com/blevesearch/bleve/analysis" + "github.com/blevesearch/bleve/v2/analysis" ) func RegisterTokenMap(name string, constructor TokenMapConstructor) { diff --git a/vendor/github.com/blevesearch/bleve/registry/tokenizer.go b/vendor/github.com/blevesearch/bleve/v2/registry/tokenizer.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/registry/tokenizer.go rename to vendor/github.com/blevesearch/bleve/v2/registry/tokenizer.go index cb9af64388..eb954287c3 100644 --- a/vendor/github.com/blevesearch/bleve/registry/tokenizer.go +++ b/vendor/github.com/blevesearch/bleve/v2/registry/tokenizer.go @@ -17,7 +17,7 @@ package registry import ( "fmt" - "github.com/blevesearch/bleve/analysis" + "github.com/blevesearch/bleve/v2/analysis" ) func RegisterTokenizer(name string, constructor TokenizerConstructor) { diff --git a/vendor/github.com/blevesearch/bleve/search.go b/vendor/github.com/blevesearch/bleve/v2/search.go similarity index 96% rename from vendor/github.com/blevesearch/bleve/search.go rename to vendor/github.com/blevesearch/bleve/v2/search.go index f67450779a..7397f566ea 100644 --- a/vendor/github.com/blevesearch/bleve/search.go +++ b/vendor/github.com/blevesearch/bleve/v2/search.go @@ -21,14 +21,14 @@ import ( "sort" "time" - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/datetime/optional" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/registry" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/collector" - "github.com/blevesearch/bleve/search/query" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/datetime/optional" + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/collector" + "github.com/blevesearch/bleve/v2/search/query" + "github.com/blevesearch/bleve/v2/size" ) var reflectStaticSizeSearchResult int @@ -600,12 +600,14 @@ func MemoryNeededForSearchResult(req *SearchRequest) uint64 { estimate += len(req.Facets) * fr.Size() } - // highlighting, store + // overhead from fields, highlighting var d document.Document if len(req.Fields) > 0 || req.Highlight != nil { - for i := 0; i < (req.Size + req.From); i++ { - estimate += (req.Size + req.From) * d.Size() + numDocsApplicable := req.Size + if numDocsApplicable > collector.PreAllocSizeSkipCap { + numDocsApplicable = collector.PreAllocSizeSkipCap } + estimate += numDocsApplicable * d.Size() } return uint64(estimate) diff --git a/vendor/github.com/blevesearch/bleve/search/collector.go b/vendor/github.com/blevesearch/bleve/v2/search/collector.go similarity index 97% rename from vendor/github.com/blevesearch/bleve/search/collector.go rename to vendor/github.com/blevesearch/bleve/v2/search/collector.go index df3ff9c5ab..38e34fe7c0 100644 --- a/vendor/github.com/blevesearch/bleve/search/collector.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/collector.go @@ -18,7 +18,7 @@ import ( "context" "time" - "github.com/blevesearch/bleve/index" + index "github.com/blevesearch/bleve_index_api" ) type Collector interface { diff --git a/vendor/github.com/blevesearch/bleve/search/collector/heap.go b/vendor/github.com/blevesearch/bleve/v2/search/collector/heap.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/search/collector/heap.go rename to vendor/github.com/blevesearch/bleve/v2/search/collector/heap.go index 05502d5dfa..9503f00603 100644 --- a/vendor/github.com/blevesearch/bleve/search/collector/heap.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/collector/heap.go @@ -17,7 +17,7 @@ package collector import ( "container/heap" - "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/v2/search" ) type collectStoreHeap struct { diff --git a/vendor/github.com/blevesearch/bleve/search/collector/list.go b/vendor/github.com/blevesearch/bleve/v2/search/collector/list.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/search/collector/list.go rename to vendor/github.com/blevesearch/bleve/v2/search/collector/list.go index f01d205c9c..20d4c9d01b 100644 --- a/vendor/github.com/blevesearch/bleve/search/collector/list.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/collector/list.go @@ -17,7 +17,7 @@ package collector import ( "container/list" - "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/v2/search" ) type collectStoreList struct { diff --git a/vendor/github.com/blevesearch/bleve/search/collector/slice.go b/vendor/github.com/blevesearch/bleve/v2/search/collector/slice.go similarity index 97% rename from vendor/github.com/blevesearch/bleve/search/collector/slice.go rename to vendor/github.com/blevesearch/bleve/v2/search/collector/slice.go index 85fe73c408..b38d9abc4f 100644 --- a/vendor/github.com/blevesearch/bleve/search/collector/slice.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/collector/slice.go @@ -14,7 +14,7 @@ package collector -import "github.com/blevesearch/bleve/search" +import "github.com/blevesearch/bleve/v2/search" type collectStoreSlice struct { slice search.DocumentMatchCollection diff --git a/vendor/github.com/blevesearch/bleve/search/collector/topn.go b/vendor/github.com/blevesearch/bleve/v2/search/collector/topn.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/search/collector/topn.go rename to vendor/github.com/blevesearch/bleve/v2/search/collector/topn.go index 8d4afb63a0..aa1d65bd17 100644 --- a/vendor/github.com/blevesearch/bleve/search/collector/topn.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/collector/topn.go @@ -20,9 +20,9 @@ import ( "strconv" "time" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeTopNCollector int @@ -68,7 +68,7 @@ type TopNCollector struct { cachedDesc []bool lowestMatchOutsideResults *search.DocumentMatch - updateFieldVisitor index.DocumentFieldTermVisitor + updateFieldVisitor index.DocValueVisitor dvReader index.DocValueReader searchAfter *search.DocumentMatch } diff --git a/vendor/github.com/blevesearch/bleve/search/explanation.go b/vendor/github.com/blevesearch/bleve/v2/search/explanation.go similarity index 97% rename from vendor/github.com/blevesearch/bleve/search/explanation.go rename to vendor/github.com/blevesearch/bleve/v2/search/explanation.go index 3b81737b50..b1ac29aa84 100644 --- a/vendor/github.com/blevesearch/bleve/search/explanation.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/explanation.go @@ -19,7 +19,7 @@ import ( "fmt" "reflect" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/size" ) var reflectStaticSizeExplanation int diff --git a/vendor/github.com/blevesearch/bleve/search/facet/benchmark_data.txt b/vendor/github.com/blevesearch/bleve/v2/search/facet/benchmark_data.txt similarity index 100% rename from vendor/github.com/blevesearch/bleve/search/facet/benchmark_data.txt rename to vendor/github.com/blevesearch/bleve/v2/search/facet/benchmark_data.txt diff --git a/vendor/github.com/blevesearch/bleve/search/facet/facet_builder_datetime.go b/vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_datetime.go similarity index 96% rename from vendor/github.com/blevesearch/bleve/search/facet/facet_builder_datetime.go rename to vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_datetime.go index c45442e4d8..a316ee4da8 100644 --- a/vendor/github.com/blevesearch/bleve/search/facet/facet_builder_datetime.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_datetime.go @@ -19,9 +19,9 @@ import ( "sort" "time" - "github.com/blevesearch/bleve/numeric" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" ) var reflectStaticSizeDateTimeFacetBuilder int diff --git a/vendor/github.com/blevesearch/bleve/search/facet/facet_builder_numeric.go b/vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_numeric.go similarity index 96% rename from vendor/github.com/blevesearch/bleve/search/facet/facet_builder_numeric.go rename to vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_numeric.go index c1692b5498..6d0c6c9d13 100644 --- a/vendor/github.com/blevesearch/bleve/search/facet/facet_builder_numeric.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_numeric.go @@ -18,9 +18,9 @@ import ( "reflect" "sort" - "github.com/blevesearch/bleve/numeric" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" ) var reflectStaticSizeNumericFacetBuilder int diff --git a/vendor/github.com/blevesearch/bleve/search/facet/facet_builder_terms.go b/vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_terms.go similarity index 96% rename from vendor/github.com/blevesearch/bleve/search/facet/facet_builder_terms.go rename to vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_terms.go index 5b5901e01c..1b378db7cd 100644 --- a/vendor/github.com/blevesearch/bleve/search/facet/facet_builder_terms.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_terms.go @@ -18,8 +18,8 @@ import ( "reflect" "sort" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" ) var reflectStaticSizeTermsFacetBuilder int diff --git a/vendor/github.com/blevesearch/bleve/search/facets_builder.go b/vendor/github.com/blevesearch/bleve/v2/search/facets_builder.go similarity index 99% rename from vendor/github.com/blevesearch/bleve/search/facets_builder.go rename to vendor/github.com/blevesearch/bleve/v2/search/facets_builder.go index 7fc0bedf30..9822257f7e 100644 --- a/vendor/github.com/blevesearch/bleve/search/facets_builder.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/facets_builder.go @@ -18,8 +18,8 @@ import ( "reflect" "sort" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeFacetsBuilder int diff --git a/vendor/github.com/blevesearch/bleve/search/highlight/format/html/html.go b/vendor/github.com/blevesearch/bleve/v2/search/highlight/format/html/html.go similarity index 92% rename from vendor/github.com/blevesearch/bleve/search/highlight/format/html/html.go rename to vendor/github.com/blevesearch/bleve/v2/search/highlight/format/html/html.go index 259a03795c..a0658d9c7f 100644 --- a/vendor/github.com/blevesearch/bleve/search/highlight/format/html/html.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/highlight/format/html/html.go @@ -17,8 +17,8 @@ package html import ( "html" - "github.com/blevesearch/bleve/registry" - "github.com/blevesearch/bleve/search/highlight" + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/search/highlight" ) const Name = "html" @@ -60,7 +60,7 @@ func (a *FragmentFormatter) Format(f *highlight.Fragment, orderedTermLocations h // start the tag rv += a.before // add the term itself - rv += string(f.Orig[termLocation.Start:termLocation.End]) + rv += html.EscapeString(string(f.Orig[termLocation.Start:termLocation.End])) // end the tag rv += a.after // update current diff --git a/vendor/github.com/blevesearch/bleve/search/highlight/fragmenter/simple/simple.go b/vendor/github.com/blevesearch/bleve/v2/search/highlight/fragmenter/simple/simple.go similarity index 97% rename from vendor/github.com/blevesearch/bleve/search/highlight/fragmenter/simple/simple.go rename to vendor/github.com/blevesearch/bleve/v2/search/highlight/fragmenter/simple/simple.go index 9c63f7fb6c..348dc1ce34 100644 --- a/vendor/github.com/blevesearch/bleve/search/highlight/fragmenter/simple/simple.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/highlight/fragmenter/simple/simple.go @@ -17,8 +17,8 @@ package simple import ( "unicode/utf8" - "github.com/blevesearch/bleve/registry" - "github.com/blevesearch/bleve/search/highlight" + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/search/highlight" ) const Name = "simple" diff --git a/vendor/github.com/blevesearch/bleve/search/highlight/highlighter.go b/vendor/github.com/blevesearch/bleve/v2/search/highlight/highlighter.go similarity index 85% rename from vendor/github.com/blevesearch/bleve/search/highlight/highlighter.go rename to vendor/github.com/blevesearch/bleve/v2/search/highlight/highlighter.go index 8077985d41..3dd9ce0531 100644 --- a/vendor/github.com/blevesearch/bleve/search/highlight/highlighter.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/highlight/highlighter.go @@ -15,8 +15,8 @@ package highlight import ( - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" ) type Fragment struct { @@ -59,6 +59,6 @@ type Highlighter interface { Separator() string SetSeparator(string) - BestFragmentInField(*search.DocumentMatch, *document.Document, string) string - BestFragmentsInField(*search.DocumentMatch, *document.Document, string, int) []string + BestFragmentInField(*search.DocumentMatch, index.Document, string) string + BestFragmentsInField(*search.DocumentMatch, index.Document, string, int) []string } diff --git a/vendor/github.com/blevesearch/bleve/search/highlight/highlighter/html/html.go b/vendor/github.com/blevesearch/bleve/v2/search/highlight/highlighter/html/html.go similarity index 78% rename from vendor/github.com/blevesearch/bleve/search/highlight/highlighter/html/html.go rename to vendor/github.com/blevesearch/bleve/v2/search/highlight/highlighter/html/html.go index 928589c477..ceb686dce5 100644 --- a/vendor/github.com/blevesearch/bleve/search/highlight/highlighter/html/html.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/highlight/highlighter/html/html.go @@ -17,11 +17,11 @@ package html import ( "fmt" - "github.com/blevesearch/bleve/registry" - "github.com/blevesearch/bleve/search/highlight" - htmlFormatter "github.com/blevesearch/bleve/search/highlight/format/html" - simpleFragmenter "github.com/blevesearch/bleve/search/highlight/fragmenter/simple" - simpleHighlighter "github.com/blevesearch/bleve/search/highlight/highlighter/simple" + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/search/highlight" + htmlFormatter "github.com/blevesearch/bleve/v2/search/highlight/format/html" + simpleFragmenter "github.com/blevesearch/bleve/v2/search/highlight/fragmenter/simple" + simpleHighlighter "github.com/blevesearch/bleve/v2/search/highlight/highlighter/simple" ) const Name = "html" diff --git a/vendor/github.com/blevesearch/bleve/search/highlight/highlighter/simple/fragment_scorer_simple.go b/vendor/github.com/blevesearch/bleve/v2/search/highlight/highlighter/simple/fragment_scorer_simple.go similarity index 93% rename from vendor/github.com/blevesearch/bleve/search/highlight/highlighter/simple/fragment_scorer_simple.go rename to vendor/github.com/blevesearch/bleve/v2/search/highlight/highlighter/simple/fragment_scorer_simple.go index 3ec4c3d28e..786e33cb3a 100644 --- a/vendor/github.com/blevesearch/bleve/search/highlight/highlighter/simple/fragment_scorer_simple.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/highlight/highlighter/simple/fragment_scorer_simple.go @@ -15,8 +15,8 @@ package simple import ( - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/highlight" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/highlight" ) // FragmentScorer will score fragments by how many diff --git a/vendor/github.com/blevesearch/bleve/search/highlight/highlighter/simple/highlighter_simple.go b/vendor/github.com/blevesearch/bleve/v2/search/highlight/highlighter/simple/highlighter_simple.go similarity index 93% rename from vendor/github.com/blevesearch/bleve/search/highlight/highlighter/simple/highlighter_simple.go rename to vendor/github.com/blevesearch/bleve/v2/search/highlight/highlighter/simple/highlighter_simple.go index 4849516bad..19949687d2 100644 --- a/vendor/github.com/blevesearch/bleve/search/highlight/highlighter/simple/highlighter_simple.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/highlight/highlighter/simple/highlighter_simple.go @@ -17,11 +17,11 @@ package simple import ( "container/heap" "fmt" + index "github.com/blevesearch/bleve_index_api" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/registry" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/highlight" + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/highlight" ) const Name = "simple" @@ -65,7 +65,7 @@ func (s *Highlighter) SetSeparator(sep string) { s.sep = sep } -func (s *Highlighter) BestFragmentInField(dm *search.DocumentMatch, doc *document.Document, field string) string { +func (s *Highlighter) BestFragmentInField(dm *search.DocumentMatch, doc index.Document, field string) string { fragments := s.BestFragmentsInField(dm, doc, field, 1) if len(fragments) > 0 { return fragments[0] @@ -73,7 +73,7 @@ func (s *Highlighter) BestFragmentInField(dm *search.DocumentMatch, doc *documen return "" } -func (s *Highlighter) BestFragmentsInField(dm *search.DocumentMatch, doc *document.Document, field string, num int) []string { +func (s *Highlighter) BestFragmentsInField(dm *search.DocumentMatch, doc index.Document, field string, num int) []string { tlm := dm.Locations[field] orderedTermLocations := highlight.OrderTermLocations(tlm) scorer := NewFragmentScorer(tlm) @@ -81,9 +81,9 @@ func (s *Highlighter) BestFragmentsInField(dm *search.DocumentMatch, doc *docume // score the fragments and put them into a priority queue ordered by score fq := make(FragmentQueue, 0) heap.Init(&fq) - for _, f := range doc.Fields { + doc.VisitFields(func(f index.Field) { if f.Name() == field { - _, ok := f.(*document.TextField) + _, ok := f.(index.TextField) if ok { termLocationsSameArrayPosition := make(highlight.TermLocations, 0) for _, otl := range orderedTermLocations { @@ -101,7 +101,7 @@ func (s *Highlighter) BestFragmentsInField(dm *search.DocumentMatch, doc *docume } } } - } + }) // now find the N best non-overlapping fragments var bestFragments []*highlight.Fragment diff --git a/vendor/github.com/blevesearch/bleve/search/highlight/term_locations.go b/vendor/github.com/blevesearch/bleve/v2/search/highlight/term_locations.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/search/highlight/term_locations.go rename to vendor/github.com/blevesearch/bleve/v2/search/highlight/term_locations.go index 6d2cb133ee..6bf385c05c 100644 --- a/vendor/github.com/blevesearch/bleve/search/highlight/term_locations.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/highlight/term_locations.go @@ -18,7 +18,7 @@ import ( "reflect" "sort" - "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/v2/search" ) type TermLocation struct { diff --git a/vendor/github.com/blevesearch/bleve/search/levenshtein.go b/vendor/github.com/blevesearch/bleve/v2/search/levenshtein.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/search/levenshtein.go rename to vendor/github.com/blevesearch/bleve/v2/search/levenshtein.go diff --git a/vendor/github.com/blevesearch/bleve/search/pool.go b/vendor/github.com/blevesearch/bleve/v2/search/pool.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/search/pool.go rename to vendor/github.com/blevesearch/bleve/v2/search/pool.go diff --git a/vendor/github.com/blevesearch/bleve/search/query/bool_field.go b/vendor/github.com/blevesearch/bleve/v2/search/query/bool_field.go similarity index 89% rename from vendor/github.com/blevesearch/bleve/search/query/bool_field.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/bool_field.go index b7b5a3d363..0272a2feb6 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/bool_field.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/bool_field.go @@ -15,10 +15,10 @@ package query import ( - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" ) type BoolFieldQuery struct { diff --git a/vendor/github.com/blevesearch/bleve/search/query/boolean.go b/vendor/github.com/blevesearch/bleve/v2/search/query/boolean.go similarity index 97% rename from vendor/github.com/blevesearch/bleve/search/query/boolean.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/boolean.go index 3cfa1d99f3..b9c504f856 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/boolean.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/boolean.go @@ -18,10 +18,10 @@ import ( "encoding/json" "fmt" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" ) type BooleanQuery struct { diff --git a/vendor/github.com/blevesearch/bleve/search/query/boost.go b/vendor/github.com/blevesearch/bleve/v2/search/query/boost.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/search/query/boost.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/boost.go diff --git a/vendor/github.com/blevesearch/bleve/search/query/conjunction.go b/vendor/github.com/blevesearch/bleve/v2/search/query/conjunction.go similarity index 93% rename from vendor/github.com/blevesearch/bleve/search/query/conjunction.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/conjunction.go index 1a7ed1bc05..7d647646e9 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/conjunction.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/conjunction.go @@ -17,10 +17,10 @@ package query import ( "encoding/json" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" ) type ConjunctionQuery struct { diff --git a/vendor/github.com/blevesearch/bleve/search/query/date_range.go b/vendor/github.com/blevesearch/bleve/v2/search/query/date_range.go similarity index 93% rename from vendor/github.com/blevesearch/bleve/search/query/date_range.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/date_range.go index 3ac0322f55..290786ddb3 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/date_range.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/date_range.go @@ -20,13 +20,13 @@ import ( "math" "time" - "github.com/blevesearch/bleve/analysis/datetime/optional" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/numeric" - "github.com/blevesearch/bleve/registry" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" + "github.com/blevesearch/bleve/v2/analysis/datetime/optional" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" ) // QueryDateTimeParser controls the default query date time parser diff --git a/vendor/github.com/blevesearch/bleve/search/query/disjunction.go b/vendor/github.com/blevesearch/bleve/v2/search/query/disjunction.go similarity index 94% rename from vendor/github.com/blevesearch/bleve/search/query/disjunction.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/disjunction.go index a1fc1439a6..50957fa672 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/disjunction.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/disjunction.go @@ -18,10 +18,10 @@ import ( "encoding/json" "fmt" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" ) type DisjunctionQuery struct { diff --git a/vendor/github.com/blevesearch/bleve/search/query/docid.go b/vendor/github.com/blevesearch/bleve/v2/search/query/docid.go similarity index 88% rename from vendor/github.com/blevesearch/bleve/search/query/docid.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/docid.go index 3b865f93ac..1d273394d6 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/docid.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/docid.go @@ -15,10 +15,10 @@ package query import ( - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" ) type DocIDQuery struct { diff --git a/vendor/github.com/blevesearch/bleve/search/query/fuzzy.go b/vendor/github.com/blevesearch/bleve/v2/search/query/fuzzy.go similarity index 91% rename from vendor/github.com/blevesearch/bleve/search/query/fuzzy.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/fuzzy.go index f18982d407..aceaa802d8 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/fuzzy.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/fuzzy.go @@ -15,10 +15,10 @@ package query import ( - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" ) type FuzzyQuery struct { diff --git a/vendor/github.com/blevesearch/bleve/search/query/geo_boundingbox.go b/vendor/github.com/blevesearch/bleve/v2/search/query/geo_boundingbox.go similarity index 93% rename from vendor/github.com/blevesearch/bleve/search/query/geo_boundingbox.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/geo_boundingbox.go index de6be4a583..be4b5a8b92 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/geo_boundingbox.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/geo_boundingbox.go @@ -18,11 +18,11 @@ import ( "encoding/json" "fmt" - "github.com/blevesearch/bleve/geo" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" ) type GeoBoundingBoxQuery struct { diff --git a/vendor/github.com/blevesearch/bleve/search/query/geo_boundingpolygon.go b/vendor/github.com/blevesearch/bleve/v2/search/query/geo_boundingpolygon.go similarity index 91% rename from vendor/github.com/blevesearch/bleve/search/query/geo_boundingpolygon.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/geo_boundingpolygon.go index 41c7f7f3ab..abb8ccd7cc 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/geo_boundingpolygon.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/geo_boundingpolygon.go @@ -18,11 +18,11 @@ import ( "encoding/json" "fmt" - "github.com/blevesearch/bleve/geo" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" ) type GeoBoundingPolygonQuery struct { diff --git a/vendor/github.com/blevesearch/bleve/search/query/geo_distance.go b/vendor/github.com/blevesearch/bleve/v2/search/query/geo_distance.go similarity index 91% rename from vendor/github.com/blevesearch/bleve/search/query/geo_distance.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/geo_distance.go index ef3aa88c99..d5174c227f 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/geo_distance.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/geo_distance.go @@ -18,11 +18,11 @@ import ( "encoding/json" "fmt" - "github.com/blevesearch/bleve/geo" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" ) type GeoDistanceQuery struct { diff --git a/vendor/github.com/blevesearch/bleve/search/query/match.go b/vendor/github.com/blevesearch/bleve/v2/search/query/match.go similarity index 95% rename from vendor/github.com/blevesearch/bleve/search/query/match.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/match.go index 36c9ee4a47..da1dc091fb 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/match.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/match.go @@ -18,9 +18,9 @@ import ( "encoding/json" "fmt" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" ) type MatchQuery struct { @@ -37,9 +37,9 @@ type MatchQueryOperator int const ( // Document must satisfy AT LEAST ONE of term searches. - MatchQueryOperatorOr = 0 + MatchQueryOperatorOr = MatchQueryOperator(0) // Document must satisfy ALL of term searches. - MatchQueryOperatorAnd = 1 + MatchQueryOperatorAnd = MatchQueryOperator(1) ) func (o MatchQueryOperator) MarshalJSON() ([]byte, error) { diff --git a/vendor/github.com/blevesearch/bleve/search/query/match_all.go b/vendor/github.com/blevesearch/bleve/v2/search/query/match_all.go similarity index 88% rename from vendor/github.com/blevesearch/bleve/search/query/match_all.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/match_all.go index 7fbe1f99db..a31f25abcf 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/match_all.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/match_all.go @@ -17,10 +17,10 @@ package query import ( "encoding/json" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" ) type MatchAllQuery struct { diff --git a/vendor/github.com/blevesearch/bleve/search/query/match_none.go b/vendor/github.com/blevesearch/bleve/v2/search/query/match_none.go similarity index 88% rename from vendor/github.com/blevesearch/bleve/search/query/match_none.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/match_none.go index dc2ea780c1..69b44186c3 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/match_none.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/match_none.go @@ -17,10 +17,10 @@ package query import ( "encoding/json" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" ) type MatchNoneQuery struct { diff --git a/vendor/github.com/blevesearch/bleve/search/query/match_phrase.go b/vendor/github.com/blevesearch/bleve/v2/search/query/match_phrase.go similarity index 94% rename from vendor/github.com/blevesearch/bleve/search/query/match_phrase.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/match_phrase.go index 51be35526f..057245fbc1 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/match_phrase.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/match_phrase.go @@ -17,10 +17,10 @@ package query import ( "fmt" - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" ) type MatchPhraseQuery struct { diff --git a/vendor/github.com/blevesearch/bleve/search/query/multi_phrase.go b/vendor/github.com/blevesearch/bleve/v2/search/query/multi_phrase.go similarity index 92% rename from vendor/github.com/blevesearch/bleve/search/query/multi_phrase.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/multi_phrase.go index 8a7c9b6ade..d75dc0c89f 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/multi_phrase.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/multi_phrase.go @@ -18,10 +18,10 @@ import ( "encoding/json" "fmt" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" ) type MultiPhraseQuery struct { diff --git a/vendor/github.com/blevesearch/bleve/search/query/numeric_range.go b/vendor/github.com/blevesearch/bleve/v2/search/query/numeric_range.go similarity index 93% rename from vendor/github.com/blevesearch/bleve/search/query/numeric_range.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/numeric_range.go index ea3f068862..a1fe7b6c16 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/numeric_range.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/numeric_range.go @@ -17,10 +17,10 @@ package query import ( "fmt" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" ) type NumericRangeQuery struct { diff --git a/vendor/github.com/blevesearch/bleve/search/query/phrase.go b/vendor/github.com/blevesearch/bleve/v2/search/query/phrase.go similarity index 91% rename from vendor/github.com/blevesearch/bleve/search/query/phrase.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/phrase.go index dff1a02d4b..d6da11853c 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/phrase.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/phrase.go @@ -18,10 +18,10 @@ import ( "encoding/json" "fmt" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" ) type PhraseQuery struct { diff --git a/vendor/github.com/blevesearch/bleve/search/query/prefix.go b/vendor/github.com/blevesearch/bleve/v2/search/query/prefix.go similarity index 89% rename from vendor/github.com/blevesearch/bleve/search/query/prefix.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/prefix.go index 4f5be2b26e..05dc40c04a 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/prefix.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/prefix.go @@ -15,10 +15,10 @@ package query import ( - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" ) type PrefixQuery struct { diff --git a/vendor/github.com/blevesearch/bleve/search/query/query.go b/vendor/github.com/blevesearch/bleve/v2/search/query/query.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/search/query/query.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/query.go index 18aca228d4..7f2781c37f 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/query.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/query.go @@ -20,9 +20,9 @@ import ( "io/ioutil" "log" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" ) var logger = log.New(ioutil.Discard, "bleve mapping ", log.LstdFlags) diff --git a/vendor/github.com/blevesearch/bleve/search/query/query_string.go b/vendor/github.com/blevesearch/bleve/v2/search/query/query_string.go similarity index 92% rename from vendor/github.com/blevesearch/bleve/search/query/query_string.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/query_string.go index ecafe6bc05..fe1680c528 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/query_string.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/query_string.go @@ -15,9 +15,9 @@ package query import ( - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" ) type QueryStringQuery struct { diff --git a/vendor/github.com/blevesearch/bleve/search/query/query_string.y b/vendor/github.com/blevesearch/bleve/v2/search/query/query_string.y similarity index 92% rename from vendor/github.com/blevesearch/bleve/search/query/query_string.y rename to vendor/github.com/blevesearch/bleve/v2/search/query/query_string.y index d3e5ac9d7e..aeec85600b 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/query_string.y +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/query_string.y @@ -28,10 +28,11 @@ tEQUAL tTILDE %type tPHRASE %type tNUMBER %type posOrNegNumber +%type fieldName %type tTILDE %type tBOOST %type searchBase -%type searchSuffix +%type searchSuffix %type searchPrefix %% @@ -111,7 +112,7 @@ tSTRING tTILDE { $$ = q } | -tSTRING tCOLON tSTRING tTILDE { +fieldName tCOLON tSTRING tTILDE { field := $1 str := $3 fuzziness, err := strconv.ParseFloat($4, 64) @@ -147,7 +148,7 @@ tPHRASE { $$ = q } | -tSTRING tCOLON tSTRING { +fieldName tCOLON tSTRING { field := $1 str := $3 logDebugGrammar("FIELD - %s STRING - %s", field, str) @@ -163,7 +164,7 @@ tSTRING tCOLON tSTRING { $$ = q } | -tSTRING tCOLON posOrNegNumber { +fieldName tCOLON posOrNegNumber { field := $1 str := $3 logDebugGrammar("FIELD - %s STRING - %s", field, str) @@ -181,7 +182,7 @@ tSTRING tCOLON posOrNegNumber { $$ = q } | -tSTRING tCOLON tPHRASE { +fieldName tCOLON tPHRASE { field := $1 phrase := $3 logDebugGrammar("FIELD - %s PHRASE - %s", field, phrase) @@ -190,7 +191,7 @@ tSTRING tCOLON tPHRASE { $$ = q } | -tSTRING tCOLON tGREATER posOrNegNumber { +fieldName tCOLON tGREATER posOrNegNumber { field := $1 min, err := strconv.ParseFloat($4, 64) if err != nil { @@ -203,7 +204,7 @@ tSTRING tCOLON tGREATER posOrNegNumber { $$ = q } | -tSTRING tCOLON tGREATER tEQUAL posOrNegNumber { +fieldName tCOLON tGREATER tEQUAL posOrNegNumber { field := $1 min, err := strconv.ParseFloat($5, 64) if err != nil { @@ -216,7 +217,7 @@ tSTRING tCOLON tGREATER tEQUAL posOrNegNumber { $$ = q } | -tSTRING tCOLON tLESS posOrNegNumber { +fieldName tCOLON tLESS posOrNegNumber { field := $1 max, err := strconv.ParseFloat($4, 64) if err != nil { @@ -229,7 +230,7 @@ tSTRING tCOLON tLESS posOrNegNumber { $$ = q } | -tSTRING tCOLON tLESS tEQUAL posOrNegNumber { +fieldName tCOLON tLESS tEQUAL posOrNegNumber { field := $1 max, err := strconv.ParseFloat($5, 64) if err != nil { @@ -242,7 +243,7 @@ tSTRING tCOLON tLESS tEQUAL posOrNegNumber { $$ = q } | -tSTRING tCOLON tGREATER tPHRASE { +fieldName tCOLON tGREATER tPHRASE { field := $1 minInclusive := false phrase := $4 @@ -257,7 +258,7 @@ tSTRING tCOLON tGREATER tPHRASE { $$ = q } | -tSTRING tCOLON tGREATER tEQUAL tPHRASE { +fieldName tCOLON tGREATER tEQUAL tPHRASE { field := $1 minInclusive := true phrase := $5 @@ -272,7 +273,7 @@ tSTRING tCOLON tGREATER tEQUAL tPHRASE { $$ = q } | -tSTRING tCOLON tLESS tPHRASE { +fieldName tCOLON tLESS tPHRASE { field := $1 maxInclusive := false phrase := $4 @@ -287,7 +288,7 @@ tSTRING tCOLON tLESS tPHRASE { $$ = q } | -tSTRING tCOLON tLESS tEQUAL tPHRASE { +fieldName tCOLON tLESS tEQUAL tPHRASE { field := $1 maxInclusive := true phrase := $5 @@ -326,3 +327,12 @@ tNUMBER { tMINUS tNUMBER { $$ = "-" + $2 }; + +fieldName: +tPHRASE { + $$ = $1 +} +| +tSTRING { + $$ = $1 +}; diff --git a/vendor/github.com/blevesearch/bleve/search/query/query_string.y.go b/vendor/github.com/blevesearch/bleve/v2/search/query/query_string.y.go similarity index 89% rename from vendor/github.com/blevesearch/bleve/search/query/query_string.y.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/query_string.y.go index ac2d322263..3a2abc1320 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/query_string.y.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/query_string.y.go @@ -1,3 +1,6 @@ +// Code generated by goyacc -o query_string.y.go query_string.y. DO NOT EDIT. + +//line query_string.y:2 package query import __yyfmt__ "fmt" @@ -54,6 +57,7 @@ var yyToknames = [...]string{ "tEQUAL", "tTILDE", } + var yyStatenames = [...]string{} const yyEofCode = 1 @@ -68,70 +72,72 @@ var yyExca = [...]int{ -1, 3, 1, 3, -2, 5, + -1, 9, + 8, 29, + -2, 8, + -1, 12, + 8, 28, + -2, 12, } -const yyNprod = 28 const yyPrivate = 57344 -var yyTokenNames []string -var yyStates []string - -const yyLast = 42 +const yyLast = 43 var yyAct = [...]int{ - - 17, 16, 18, 23, 22, 30, 3, 21, 19, 20, - 29, 26, 22, 22, 1, 21, 21, 15, 28, 25, - 24, 27, 34, 14, 22, 13, 31, 21, 32, 33, - 22, 9, 11, 21, 5, 6, 2, 10, 4, 12, - 7, 8, + 18, 17, 19, 24, 23, 15, 31, 22, 20, 21, + 30, 27, 23, 23, 3, 22, 22, 14, 29, 26, + 16, 25, 28, 35, 33, 23, 23, 32, 22, 22, + 34, 9, 12, 1, 5, 6, 2, 11, 4, 13, + 7, 8, 10, } + var yyPact = [...]int{ - - 28, -1000, -1000, 28, 27, -1000, -1000, -1000, 16, 9, - -1000, -1000, -1000, -1000, -1000, -3, -11, -1000, -1000, 6, - 5, -1000, -5, -1000, -1000, 23, -1000, -1000, 17, -1000, - -1000, -1000, -1000, -1000, -1000, + 28, -1000, -1000, 28, 27, -1000, -1000, -1000, 8, -9, + 12, -1000, -1000, -1000, -1000, -1000, -3, -11, -1000, -1000, + 6, 5, -1000, -4, -1000, -1000, 19, -1000, -1000, 18, + -1000, -1000, -1000, -1000, -1000, -1000, } + var yyPgo = [...]int{ - - 0, 0, 41, 39, 38, 14, 36, 6, + 0, 0, 42, 41, 39, 38, 33, 36, 14, } + var yyR1 = [...]int{ - - 0, 5, 6, 6, 7, 4, 4, 4, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 3, 3, 1, 1, + 0, 6, 7, 7, 8, 5, 5, 5, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 4, 4, 1, 1, 2, 2, } -var yyR2 = [...]int{ +var yyR2 = [...]int{ 0, 1, 2, 1, 3, 0, 1, 1, 1, 2, 4, 1, 1, 3, 3, 3, 4, 5, 4, 5, - 4, 5, 4, 5, 0, 1, 1, 2, + 4, 5, 4, 5, 0, 1, 1, 2, 1, 1, } + var yyChk = [...]int{ - - -1000, -5, -6, -7, -4, 6, 7, -6, -2, 4, - 10, 5, -3, 9, 14, 8, 4, -1, 5, 11, - 12, 10, 7, 14, -1, 13, 5, -1, 13, 5, - 10, -1, 5, -1, 5, + -1000, -6, -7, -8, -5, 6, 7, -7, -3, 4, + -2, 10, 5, -4, 9, 14, 8, 4, -1, 5, + 11, 12, 10, 7, 14, -1, 13, 5, -1, 13, + 5, 10, -1, 5, -1, 5, } + var yyDef = [...]int{ - - 5, -2, 1, -2, 0, 6, 7, 2, 24, 8, - 11, 12, 4, 25, 9, 0, 13, 14, 15, 0, - 0, 26, 0, 10, 16, 0, 20, 18, 0, 22, - 27, 17, 21, 19, 23, + 5, -2, 1, -2, 0, 6, 7, 2, 24, -2, + 0, 11, -2, 4, 25, 9, 0, 13, 14, 15, + 0, 0, 26, 0, 10, 16, 0, 20, 18, 0, + 22, 27, 17, 21, 19, 23, } -var yyTok1 = [...]int{ +var yyTok1 = [...]int{ 1, } -var yyTok2 = [...]int{ +var yyTok2 = [...]int{ 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, } + var yyTok3 = [...]int{ 0, } @@ -475,25 +481,25 @@ yydefault: case 1: yyDollar = yyS[yypt-1 : yypt+1] - //line query_string.y:40 +//line query_string.y:41 { logDebugGrammar("INPUT") } case 2: yyDollar = yyS[yypt-2 : yypt+1] - //line query_string.y:45 +//line query_string.y:46 { logDebugGrammar("SEARCH PARTS") } case 3: yyDollar = yyS[yypt-1 : yypt+1] - //line query_string.y:49 +//line query_string.y:50 { logDebugGrammar("SEARCH PART") } case 4: yyDollar = yyS[yypt-3 : yypt+1] - //line query_string.y:54 +//line query_string.y:55 { query := yyDollar[2].q if yyDollar[3].pf != nil { @@ -512,27 +518,27 @@ yydefault: } case 5: yyDollar = yyS[yypt-0 : yypt+1] - //line query_string.y:73 +//line query_string.y:74 { yyVAL.n = queryShould } case 6: yyDollar = yyS[yypt-1 : yypt+1] - //line query_string.y:77 +//line query_string.y:78 { logDebugGrammar("PLUS") yyVAL.n = queryMust } case 7: yyDollar = yyS[yypt-1 : yypt+1] - //line query_string.y:82 +//line query_string.y:83 { logDebugGrammar("MINUS") yyVAL.n = queryMustNot } case 8: yyDollar = yyS[yypt-1 : yypt+1] - //line query_string.y:88 +//line query_string.y:89 { str := yyDollar[1].s logDebugGrammar("STRING - %s", str) @@ -548,7 +554,7 @@ yydefault: } case 9: yyDollar = yyS[yypt-2 : yypt+1] - //line query_string.y:102 +//line query_string.y:103 { str := yyDollar[1].s fuzziness, err := strconv.ParseFloat(yyDollar[2].s, 64) @@ -562,7 +568,7 @@ yydefault: } case 10: yyDollar = yyS[yypt-4 : yypt+1] - //line query_string.y:114 +//line query_string.y:115 { field := yyDollar[1].s str := yyDollar[3].s @@ -578,7 +584,7 @@ yydefault: } case 11: yyDollar = yyS[yypt-1 : yypt+1] - //line query_string.y:128 +//line query_string.y:129 { str := yyDollar[1].s logDebugGrammar("STRING - %s", str) @@ -595,7 +601,7 @@ yydefault: } case 12: yyDollar = yyS[yypt-1 : yypt+1] - //line query_string.y:143 +//line query_string.y:144 { phrase := yyDollar[1].s logDebugGrammar("PHRASE - %s", phrase) @@ -604,7 +610,7 @@ yydefault: } case 13: yyDollar = yyS[yypt-3 : yypt+1] - //line query_string.y:150 +//line query_string.y:151 { field := yyDollar[1].s str := yyDollar[3].s @@ -622,7 +628,7 @@ yydefault: } case 14: yyDollar = yyS[yypt-3 : yypt+1] - //line query_string.y:166 +//line query_string.y:167 { field := yyDollar[1].s str := yyDollar[3].s @@ -642,7 +648,7 @@ yydefault: } case 15: yyDollar = yyS[yypt-3 : yypt+1] - //line query_string.y:184 +//line query_string.y:185 { field := yyDollar[1].s phrase := yyDollar[3].s @@ -653,7 +659,7 @@ yydefault: } case 16: yyDollar = yyS[yypt-4 : yypt+1] - //line query_string.y:193 +//line query_string.y:194 { field := yyDollar[1].s min, err := strconv.ParseFloat(yyDollar[4].s, 64) @@ -668,7 +674,7 @@ yydefault: } case 17: yyDollar = yyS[yypt-5 : yypt+1] - //line query_string.y:206 +//line query_string.y:207 { field := yyDollar[1].s min, err := strconv.ParseFloat(yyDollar[5].s, 64) @@ -683,7 +689,7 @@ yydefault: } case 18: yyDollar = yyS[yypt-4 : yypt+1] - //line query_string.y:219 +//line query_string.y:220 { field := yyDollar[1].s max, err := strconv.ParseFloat(yyDollar[4].s, 64) @@ -698,7 +704,7 @@ yydefault: } case 19: yyDollar = yyS[yypt-5 : yypt+1] - //line query_string.y:232 +//line query_string.y:233 { field := yyDollar[1].s max, err := strconv.ParseFloat(yyDollar[5].s, 64) @@ -713,7 +719,7 @@ yydefault: } case 20: yyDollar = yyS[yypt-4 : yypt+1] - //line query_string.y:245 +//line query_string.y:246 { field := yyDollar[1].s minInclusive := false @@ -730,7 +736,7 @@ yydefault: } case 21: yyDollar = yyS[yypt-5 : yypt+1] - //line query_string.y:260 +//line query_string.y:261 { field := yyDollar[1].s minInclusive := true @@ -747,7 +753,7 @@ yydefault: } case 22: yyDollar = yyS[yypt-4 : yypt+1] - //line query_string.y:275 +//line query_string.y:276 { field := yyDollar[1].s maxInclusive := false @@ -764,7 +770,7 @@ yydefault: } case 23: yyDollar = yyS[yypt-5 : yypt+1] - //line query_string.y:290 +//line query_string.y:291 { field := yyDollar[1].s maxInclusive := true @@ -781,13 +787,13 @@ yydefault: } case 24: yyDollar = yyS[yypt-0 : yypt+1] - //line query_string.y:306 +//line query_string.y:307 { yyVAL.pf = nil } case 25: yyDollar = yyS[yypt-1 : yypt+1] - //line query_string.y:310 +//line query_string.y:311 { yyVAL.pf = nil boost, err := strconv.ParseFloat(yyDollar[1].s, 64) @@ -800,16 +806,28 @@ yydefault: } case 26: yyDollar = yyS[yypt-1 : yypt+1] - //line query_string.y:322 +//line query_string.y:323 { yyVAL.s = yyDollar[1].s } case 27: yyDollar = yyS[yypt-2 : yypt+1] - //line query_string.y:326 +//line query_string.y:327 { yyVAL.s = "-" + yyDollar[2].s } + case 28: + yyDollar = yyS[yypt-1 : yypt+1] +//line query_string.y:332 + { + yyVAL.s = yyDollar[1].s + } + case 29: + yyDollar = yyS[yypt-1 : yypt+1] +//line query_string.y:336 + { + yyVAL.s = yyDollar[1].s + } } goto yystack /* stack new state and value */ } diff --git a/vendor/github.com/blevesearch/bleve/search/query/query_string_lex.go b/vendor/github.com/blevesearch/bleve/v2/search/query/query_string_lex.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/search/query/query_string_lex.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/query_string_lex.go diff --git a/vendor/github.com/blevesearch/bleve/search/query/query_string_parser.go b/vendor/github.com/blevesearch/bleve/v2/search/query/query_string_parser.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/search/query/query_string_parser.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/query_string_parser.go diff --git a/vendor/github.com/blevesearch/bleve/search/query/regexp.go b/vendor/github.com/blevesearch/bleve/v2/search/query/regexp.go similarity index 92% rename from vendor/github.com/blevesearch/bleve/search/query/regexp.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/regexp.go index 0c87a6f92e..ba744ec15a 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/regexp.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/regexp.go @@ -17,10 +17,10 @@ package query import ( "strings" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" ) type RegexpQuery struct { diff --git a/vendor/github.com/blevesearch/bleve/search/query/term.go b/vendor/github.com/blevesearch/bleve/v2/search/query/term.go similarity index 89% rename from vendor/github.com/blevesearch/bleve/search/query/term.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/term.go index 2eeb5a37a4..82958bb02f 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/term.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/term.go @@ -15,10 +15,10 @@ package query import ( - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" ) type TermQuery struct { diff --git a/vendor/github.com/blevesearch/bleve/search/query/term_range.go b/vendor/github.com/blevesearch/bleve/v2/search/query/term_range.go similarity index 93% rename from vendor/github.com/blevesearch/bleve/search/query/term_range.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/term_range.go index 8f8ca8444f..3edfa6954a 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/term_range.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/term_range.go @@ -17,10 +17,10 @@ package query import ( "fmt" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" ) type TermRangeQuery struct { diff --git a/vendor/github.com/blevesearch/bleve/search/query/wildcard.go b/vendor/github.com/blevesearch/bleve/v2/search/query/wildcard.go similarity index 92% rename from vendor/github.com/blevesearch/bleve/search/query/wildcard.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/wildcard.go index 747dfe76ff..7713a9acb5 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/wildcard.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/wildcard.go @@ -17,10 +17,10 @@ package query import ( "strings" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" ) var wildcardRegexpReplacer = strings.NewReplacer( diff --git a/vendor/github.com/blevesearch/bleve/search/scorer/scorer_conjunction.go b/vendor/github.com/blevesearch/bleve/v2/search/scorer/scorer_conjunction.go similarity index 96% rename from vendor/github.com/blevesearch/bleve/search/scorer/scorer_conjunction.go rename to vendor/github.com/blevesearch/bleve/v2/search/scorer/scorer_conjunction.go index 48cdf3ae90..f3c81a78c8 100644 --- a/vendor/github.com/blevesearch/bleve/search/scorer/scorer_conjunction.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/scorer/scorer_conjunction.go @@ -17,8 +17,8 @@ package scorer import ( "reflect" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" ) var reflectStaticSizeConjunctionQueryScorer int diff --git a/vendor/github.com/blevesearch/bleve/search/scorer/scorer_constant.go b/vendor/github.com/blevesearch/bleve/v2/search/scorer/scorer_constant.go similarity index 95% rename from vendor/github.com/blevesearch/bleve/search/scorer/scorer_constant.go rename to vendor/github.com/blevesearch/bleve/v2/search/scorer/scorer_constant.go index dc10fdaa4e..fc36fd5bfa 100644 --- a/vendor/github.com/blevesearch/bleve/search/scorer/scorer_constant.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/scorer/scorer_constant.go @@ -18,9 +18,9 @@ import ( "fmt" "reflect" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeConstantScorer int diff --git a/vendor/github.com/blevesearch/bleve/search/scorer/scorer_disjunction.go b/vendor/github.com/blevesearch/bleve/v2/search/scorer/scorer_disjunction.go similarity index 96% rename from vendor/github.com/blevesearch/bleve/search/scorer/scorer_disjunction.go rename to vendor/github.com/blevesearch/bleve/v2/search/scorer/scorer_disjunction.go index 7a955e168e..054e76fd42 100644 --- a/vendor/github.com/blevesearch/bleve/search/scorer/scorer_disjunction.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/scorer/scorer_disjunction.go @@ -18,8 +18,8 @@ import ( "fmt" "reflect" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" ) var reflectStaticSizeDisjunctionQueryScorer int diff --git a/vendor/github.com/blevesearch/bleve/search/scorer/scorer_term.go b/vendor/github.com/blevesearch/bleve/v2/search/scorer/scorer_term.go similarity index 97% rename from vendor/github.com/blevesearch/bleve/search/scorer/scorer_term.go rename to vendor/github.com/blevesearch/bleve/v2/search/scorer/scorer_term.go index 718de2ea5e..ca268648b3 100644 --- a/vendor/github.com/blevesearch/bleve/search/scorer/scorer_term.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/scorer/scorer_term.go @@ -19,9 +19,9 @@ import ( "math" "reflect" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeTermQueryScorer int diff --git a/vendor/github.com/blevesearch/bleve/search/scorer/sqrt_cache.go b/vendor/github.com/blevesearch/bleve/v2/search/scorer/sqrt_cache.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/search/scorer/sqrt_cache.go rename to vendor/github.com/blevesearch/bleve/v2/search/scorer/sqrt_cache.go diff --git a/vendor/github.com/blevesearch/bleve/search/search.go b/vendor/github.com/blevesearch/bleve/v2/search/search.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/search/search.go rename to vendor/github.com/blevesearch/bleve/v2/search/search.go index 8ed23de454..d2dd33712e 100644 --- a/vendor/github.com/blevesearch/bleve/search/search.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/search.go @@ -19,8 +19,8 @@ import ( "reflect" "sort" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeDocumentMatch int @@ -270,7 +270,7 @@ func (dm *DocumentMatch) Complete(prealloc []Location) []Location { var needsDedupe bool for i, ftl := range dm.FieldTermLocations { - if lastField != ftl.Field { + if i == 0 || lastField != ftl.Field { lastField = ftl.Field if dm.Locations == nil { diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/ordered_searchers_list.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/ordered_searchers_list.go similarity index 95% rename from vendor/github.com/blevesearch/bleve/search/searcher/ordered_searchers_list.go rename to vendor/github.com/blevesearch/bleve/v2/search/searcher/ordered_searchers_list.go index 536c593cd7..f3e646e9da 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/ordered_searchers_list.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/ordered_searchers_list.go @@ -15,7 +15,7 @@ package searcher import ( - "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/v2/search" ) type OrderedSearcherList []search.Searcher diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_boolean.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_boolean.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/search/searcher/search_boolean.go rename to vendor/github.com/blevesearch/bleve/v2/search/searcher/search_boolean.go index 7f0bfa4246..ef9093c20f 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_boolean.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_boolean.go @@ -18,10 +18,10 @@ import ( "math" "reflect" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/scorer" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/scorer" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeBooleanSearcher int diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_conjunction.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_conjunction.go similarity index 97% rename from vendor/github.com/blevesearch/bleve/search/searcher/search_conjunction.go rename to vendor/github.com/blevesearch/bleve/v2/search/searcher/search_conjunction.go index ac737bccd0..5fe59b900f 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_conjunction.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_conjunction.go @@ -19,10 +19,10 @@ import ( "reflect" "sort" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/scorer" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/scorer" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeConjunctionSearcher int diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_disjunction.go similarity index 97% rename from vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction.go rename to vendor/github.com/blevesearch/bleve/v2/search/searcher/search_disjunction.go index f47da27c4f..4cee468419 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_disjunction.go @@ -16,8 +16,8 @@ package searcher import ( "fmt" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" ) // DisjunctionMaxClauseCount is a compile time setting that applications can diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction_heap.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_disjunction_heap.go similarity index 97% rename from vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction_heap.go rename to vendor/github.com/blevesearch/bleve/v2/search/searcher/search_disjunction_heap.go index 7f0a5a00e3..bf945976d2 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction_heap.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_disjunction_heap.go @@ -20,10 +20,10 @@ import ( "math" "reflect" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/scorer" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/scorer" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeDisjunctionHeapSearcher int diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction_slice.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_disjunction_slice.go similarity index 97% rename from vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction_slice.go rename to vendor/github.com/blevesearch/bleve/v2/search/searcher/search_disjunction_slice.go index dc566ade57..79fee9f4da 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction_slice.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_disjunction_slice.go @@ -19,10 +19,10 @@ import ( "reflect" "sort" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/scorer" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/scorer" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeDisjunctionSliceSearcher int diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_docid.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_docid.go similarity index 93% rename from vendor/github.com/blevesearch/bleve/search/searcher/search_docid.go rename to vendor/github.com/blevesearch/bleve/v2/search/searcher/search_docid.go index 3b258a580a..2d90ae166b 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_docid.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_docid.go @@ -17,10 +17,10 @@ package searcher import ( "reflect" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/scorer" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/scorer" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeDocIDSearcher int diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_filter.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_filter.go similarity index 95% rename from vendor/github.com/blevesearch/bleve/search/searcher/search_filter.go rename to vendor/github.com/blevesearch/bleve/v2/search/searcher/search_filter.go index 7c95fb41c6..9cab0f78bd 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_filter.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_filter.go @@ -17,9 +17,9 @@ package searcher import ( "reflect" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeFilteringSearcher int diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_fuzzy.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_fuzzy.go similarity index 97% rename from vendor/github.com/blevesearch/bleve/search/searcher/search_fuzzy.go rename to vendor/github.com/blevesearch/bleve/v2/search/searcher/search_fuzzy.go index aca8a7d9fa..aab67010ca 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_fuzzy.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_fuzzy.go @@ -17,8 +17,8 @@ package searcher import ( "fmt" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" ) var MaxFuzziness = 2 diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_geoboundingbox.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_geoboundingbox.go similarity index 91% rename from vendor/github.com/blevesearch/bleve/search/searcher/search_geoboundingbox.go rename to vendor/github.com/blevesearch/bleve/v2/search/searcher/search_geoboundingbox.go index 76157f01a1..a23151466e 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_geoboundingbox.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_geoboundingbox.go @@ -15,11 +15,11 @@ package searcher import ( - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/geo" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/numeric" - "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" ) type filterFunc func(key []byte) bool @@ -100,6 +100,7 @@ func NewGeoBoundingBoxSearcher(indexReader index.IndexReader, minLon, minLat, var geoMaxShift = document.GeoPrecisionStep * 4 var geoDetailLevel = ((geo.GeoBits << 1) - geoMaxShift) / 2 + type closeFunc func() error func ComputeGeoRange(term uint64, shift uint, @@ -156,16 +157,16 @@ func buildIsIndexedFunc(indexReader index.IndexReader, field string) (isIndexed } } else if indexReader != nil { isIndexed = func(term []byte) bool { - reader, err := indexReader.TermFieldReader(term, field, false, false, false) - if err != nil || reader == nil { - return false - } - if reader.Count() == 0 { - _ = reader.Close() - return false - } + reader, err := indexReader.TermFieldReader(term, field, false, false, false) + if err != nil || reader == nil { + return false + } + if reader.Count() == 0 { _ = reader.Close() - return true + return false + } + _ = reader.Close() + return true } } else { @@ -209,12 +210,12 @@ func buildRectFilter(dvReader index.DocValueReader, field string, } type geoRangeCompute struct { - preallocBytesLen int - preallocBytes []byte + preallocBytesLen int + preallocBytes []byte sminLon, sminLat, smaxLon, smaxLat float64 - checkBoundaries bool - onBoundary, notOnBoundary [][]byte - isIndexed func(term []byte) bool + checkBoundaries bool + onBoundary, notOnBoundary [][]byte + isIndexed func(term []byte) bool } func (grc *geoRangeCompute) makePrefixCoded(in int64, shift uint) (rv numeric.PrefixCoded) { @@ -269,4 +270,4 @@ func (grc *geoRangeCompute) relateAndRecurse(start, end uint64, res uint) { grc.sminLon, grc.sminLat, grc.smaxLon, grc.smaxLat) { grc.computeGeoRange(start, res-1) } -} \ No newline at end of file +} diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_geopointdistance.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_geopointdistance.go similarity index 95% rename from vendor/github.com/blevesearch/bleve/search/searcher/search_geopointdistance.go rename to vendor/github.com/blevesearch/bleve/v2/search/searcher/search_geopointdistance.go index b6f2932445..c62a2a5b79 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_geopointdistance.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_geopointdistance.go @@ -15,10 +15,10 @@ package searcher import ( - "github.com/blevesearch/bleve/geo" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/numeric" - "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" ) func NewGeoPointDistanceSearcher(indexReader index.IndexReader, centerLon, diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_geopolygon.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_geopolygon.go similarity index 95% rename from vendor/github.com/blevesearch/bleve/search/searcher/search_geopolygon.go rename to vendor/github.com/blevesearch/bleve/v2/search/searcher/search_geopolygon.go index 5f16aa8d26..9f7e61d9eb 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_geopolygon.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_geopolygon.go @@ -16,10 +16,10 @@ package searcher import ( "fmt" - "github.com/blevesearch/bleve/geo" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/numeric" - "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" "math" ) diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_match_all.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_match_all.go similarity index 93% rename from vendor/github.com/blevesearch/bleve/search/searcher/search_match_all.go rename to vendor/github.com/blevesearch/bleve/v2/search/searcher/search_match_all.go index bb66401229..db8306eee5 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_match_all.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_match_all.go @@ -17,10 +17,10 @@ package searcher import ( "reflect" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/scorer" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/scorer" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeMatchAllSearcher int diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_match_none.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_match_none.go similarity index 93% rename from vendor/github.com/blevesearch/bleve/search/searcher/search_match_none.go rename to vendor/github.com/blevesearch/bleve/v2/search/searcher/search_match_none.go index a345e17f77..b7f76941ec 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_match_none.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_match_none.go @@ -17,9 +17,9 @@ package searcher import ( "reflect" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeMatchNoneSearcher int diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_multi_term.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_multi_term.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/search/searcher/search_multi_term.go rename to vendor/github.com/blevesearch/bleve/v2/search/searcher/search_multi_term.go index 70a2fa38c0..523bf4b557 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_multi_term.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_multi_term.go @@ -16,8 +16,8 @@ package searcher import ( "fmt" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" ) func NewMultiTermSearcher(indexReader index.IndexReader, terms []string, diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_numeric_range.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_numeric_range.go similarity index 91% rename from vendor/github.com/blevesearch/bleve/search/searcher/search_numeric_range.go rename to vendor/github.com/blevesearch/bleve/v2/search/searcher/search_numeric_range.go index 48d6226e11..6ab5147bee 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_numeric_range.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_numeric_range.go @@ -19,9 +19,9 @@ import ( "math" "sort" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/numeric" - "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" ) func NewNumericRangeSearcher(indexReader index.IndexReader, @@ -106,24 +106,6 @@ func NewNumericRangeSearcher(indexReader index.IndexReader, func filterCandidateTerms(indexReader index.IndexReader, terms [][]byte, field string) (rv [][]byte, err error) { - if ir, ok := indexReader.(index.IndexReaderOnly); ok { - fieldDict, err := ir.FieldDictOnly(field, terms, false) - if err != nil { - return nil, err - } - // enumerate the terms (no need to check them again) - tfd, err := fieldDict.Next() - for err == nil && tfd != nil { - rv = append(rv, []byte(tfd.Term)) - tfd, err = fieldDict.Next() - } - if cerr := fieldDict.Close(); cerr != nil && err == nil { - err = cerr - } - - return rv, err - } - fieldDict, err := indexReader.FieldDictRange(field, terms[0], terms[len(terms)-1]) if err != nil { return nil, err diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_phrase.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_phrase.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/search/searcher/search_phrase.go rename to vendor/github.com/blevesearch/bleve/v2/search/searcher/search_phrase.go index 51b7e5bd8a..c262fd9145 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_phrase.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_phrase.go @@ -19,9 +19,9 @@ import ( "math" "reflect" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizePhraseSearcher int diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_regexp.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_regexp.go similarity index 86% rename from vendor/github.com/blevesearch/bleve/search/searcher/search_regexp.go rename to vendor/github.com/blevesearch/bleve/v2/search/searcher/search_regexp.go index 11a44f159f..81b1cf734a 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_regexp.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_regexp.go @@ -17,10 +17,21 @@ package searcher import ( "regexp" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" ) +// The Regexp interface defines the subset of the regexp.Regexp API +// methods that are used by bleve indexes, allowing callers to pass in +// alternate implementations. +type Regexp interface { + FindStringIndex(s string) (loc []int) + + LiteralPrefix() (prefix string, complete bool) + + String() string +} + // NewRegexpStringSearcher is similar to NewRegexpSearcher, but // additionally optimizes for index readers that handle regexp's. func NewRegexpStringSearcher(indexReader index.IndexReader, pattern string, @@ -66,7 +77,7 @@ func NewRegexpStringSearcher(indexReader index.IndexReader, pattern string, // matching the entire term. The provided regexp SHOULD NOT start with ^ // or end with $ as this can intefere with the implementation. Separately, // matches will be checked to ensure they match the entire term. -func NewRegexpSearcher(indexReader index.IndexReader, pattern index.Regexp, +func NewRegexpSearcher(indexReader index.IndexReader, pattern Regexp, field string, boost float64, options search.SearcherOptions) ( search.Searcher, error) { var candidateTerms []string @@ -89,7 +100,7 @@ func NewRegexpSearcher(indexReader index.IndexReader, pattern index.Regexp, } func findRegexpCandidateTerms(indexReader index.IndexReader, - pattern index.Regexp, field, prefixTerm string) (rv []string, err error) { + pattern Regexp, field, prefixTerm string) (rv []string, err error) { rv = make([]string, 0) var fieldDict index.FieldDict if len(prefixTerm) > 0 { diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_term.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_term.go similarity index 95% rename from vendor/github.com/blevesearch/bleve/search/searcher/search_term.go rename to vendor/github.com/blevesearch/bleve/v2/search/searcher/search_term.go index e07d25333a..55c18d1634 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_term.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_term.go @@ -17,10 +17,10 @@ package searcher import ( "reflect" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/scorer" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/scorer" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeTermSearcher int diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_term_prefix.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_term_prefix.go similarity index 93% rename from vendor/github.com/blevesearch/bleve/search/searcher/search_term_prefix.go rename to vendor/github.com/blevesearch/bleve/v2/search/searcher/search_term_prefix.go index 2a8f22cff1..a01b18690a 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_term_prefix.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_term_prefix.go @@ -15,8 +15,8 @@ package searcher import ( - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" ) func NewTermPrefixSearcher(indexReader index.IndexReader, prefix string, diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_term_range.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_term_range.go similarity index 96% rename from vendor/github.com/blevesearch/bleve/search/searcher/search_term_range.go rename to vendor/github.com/blevesearch/bleve/v2/search/searcher/search_term_range.go index 90be1e11a2..5ef58f76f7 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_term_range.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_term_range.go @@ -15,8 +15,8 @@ package searcher import ( - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" ) func NewTermRangeSearcher(indexReader index.IndexReader, diff --git a/vendor/github.com/blevesearch/bleve/search/sort.go b/vendor/github.com/blevesearch/bleve/v2/search/sort.go similarity index 99% rename from vendor/github.com/blevesearch/bleve/search/sort.go rename to vendor/github.com/blevesearch/bleve/v2/search/sort.go index dca422ebd4..9ed9a78c01 100644 --- a/vendor/github.com/blevesearch/bleve/search/sort.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/sort.go @@ -21,12 +21,13 @@ import ( "math" "sort" "strings" + "unicode/utf8" - "github.com/blevesearch/bleve/geo" - "github.com/blevesearch/bleve/numeric" + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/numeric" ) -var HighTerm = strings.Repeat(string([]byte{0xff}), 10) +var HighTerm = strings.Repeat(string(utf8.MaxRune), 3) var LowTerm = string([]byte{0x00}) type SearchSort interface { diff --git a/vendor/github.com/blevesearch/bleve/search/util.go b/vendor/github.com/blevesearch/bleve/v2/search/util.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/search/util.go rename to vendor/github.com/blevesearch/bleve/v2/search/util.go diff --git a/vendor/github.com/blevesearch/bleve/size/sizes.go b/vendor/github.com/blevesearch/bleve/v2/size/sizes.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/size/sizes.go rename to vendor/github.com/blevesearch/bleve/v2/size/sizes.go diff --git a/vendor/github.com/blevesearch/bleve_index_api/.golangci.yml b/vendor/github.com/blevesearch/bleve_index_api/.golangci.yml new file mode 100644 index 0000000000..a00f6c57e4 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve_index_api/.golangci.yml @@ -0,0 +1,37 @@ +linters: + # please, do not use `enable-all`: it's deprecated and will be removed soon. + # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint + disable-all: true + enable: + - bodyclose + - deadcode + - depguard + - dogsled + - dupl + - errcheck + - goconst + - gocritic + - gocyclo + - gofmt + - goimports + - gomnd + - goprintffuncname + - gosimple + - govet + - ineffassign + - interfacer + - lll + - misspell + - nakedret + - nolintlint + - rowserrcheck + - scopelint + - staticcheck + - structcheck + - stylecheck + - typecheck + - unconvert + - unparam + - unused + - varcheck + - whitespace diff --git a/vendor/github.com/blevesearch/zap/v11/LICENSE b/vendor/github.com/blevesearch/bleve_index_api/LICENSE similarity index 100% rename from vendor/github.com/blevesearch/zap/v11/LICENSE rename to vendor/github.com/blevesearch/bleve_index_api/LICENSE diff --git a/vendor/github.com/blevesearch/bleve_index_api/README.md b/vendor/github.com/blevesearch/bleve_index_api/README.md new file mode 100644 index 0000000000..46daa68322 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve_index_api/README.md @@ -0,0 +1,11 @@ +# Bleve Index API + +[![PkgGoDev](https://pkg.go.dev/badge/github.com/blevesearch/bleve_index_api)](https://pkg.go.dev/github.com/blevesearch/bleve_index_api) +[![Tests](https://github.com/blevesearch/bleve_index_api/workflows/Tests/badge.svg?branch=master&event=push)](https://github.com/blevesearch/bleve_index_api/actions?query=workflow%3ATests+event%3Apush+branch%3Amaster) +[![Lint](https://github.com/blevesearch/bleve_index_api/workflows/Lint/badge.svg?branch=master&event=push)](https://github.com/blevesearch/bleve_index_api/actions?query=workflow%3ALint+event%3Apush+branch%3Amaster) + +Bleve supports a pluggable Index interface. + +By placing these interfaces in their own, *hopefully* slowly evolving module, it frees up Bleve and the underlying index to each introduce new major versions without interfering with one another. + +With that in mind, we anticipate introducing non-breaking changes only to this module, and keeping the major version at 1.x for some time. \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve_index_api/analysis.go b/vendor/github.com/blevesearch/bleve_index_api/analysis.go new file mode 100644 index 0000000000..5ab616c1a0 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve_index_api/analysis.go @@ -0,0 +1,53 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package index + +type AnalysisWork func() + +type AnalysisQueue struct { + queue chan AnalysisWork + done chan struct{} +} + +func (q *AnalysisQueue) Queue(work AnalysisWork) { + q.queue <- work +} + +func (q *AnalysisQueue) Close() { + close(q.done) +} + +func NewAnalysisQueue(numWorkers int) *AnalysisQueue { + rv := AnalysisQueue{ + queue: make(chan AnalysisWork), + done: make(chan struct{}), + } + for i := 0; i < numWorkers; i++ { + go AnalysisWorker(rv) + } + return &rv +} + +func AnalysisWorker(q AnalysisQueue) { + // read work off the queue + for { + select { + case <-q.done: + return + case w := <-q.queue: + w() + } + } +} diff --git a/vendor/github.com/blevesearch/bleve_index_api/batch.go b/vendor/github.com/blevesearch/bleve_index_api/batch.go new file mode 100644 index 0000000000..ff1eaf6c6e --- /dev/null +++ b/vendor/github.com/blevesearch/bleve_index_api/batch.go @@ -0,0 +1,101 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package index + +import "fmt" + +type BatchCallback func(error) + +type Batch struct { + IndexOps map[string]Document + InternalOps map[string][]byte + persistedCallback BatchCallback +} + +func NewBatch() *Batch { + return &Batch{ + IndexOps: make(map[string]Document), + InternalOps: make(map[string][]byte), + } +} + +func (b *Batch) Update(doc Document) { + b.IndexOps[doc.ID()] = doc +} + +func (b *Batch) Delete(id string) { + b.IndexOps[id] = nil +} + +func (b *Batch) SetInternal(key, val []byte) { + b.InternalOps[string(key)] = val +} + +func (b *Batch) DeleteInternal(key []byte) { + b.InternalOps[string(key)] = nil +} + +func (b *Batch) SetPersistedCallback(f BatchCallback) { + b.persistedCallback = f +} + +func (b *Batch) PersistedCallback() BatchCallback { + return b.persistedCallback +} + +func (b *Batch) String() string { + rv := fmt.Sprintf("Batch (%d ops, %d internal ops)\n", len(b.IndexOps), len(b.InternalOps)) + for k, v := range b.IndexOps { + if v != nil { + rv += fmt.Sprintf("\tINDEX - '%s'\n", k) + } else { + rv += fmt.Sprintf("\tDELETE - '%s'\n", k) + } + } + for k, v := range b.InternalOps { + if v != nil { + rv += fmt.Sprintf("\tSET INTERNAL - '%s'\n", k) + } else { + rv += fmt.Sprintf("\tDELETE INTERNAL - '%s'\n", k) + } + } + return rv +} + +func (b *Batch) Reset() { + b.IndexOps = make(map[string]Document) + b.InternalOps = make(map[string][]byte) + b.persistedCallback = nil +} + +func (b *Batch) Merge(o *Batch) { + for k, v := range o.IndexOps { + b.IndexOps[k] = v + } + for k, v := range o.InternalOps { + b.InternalOps[k] = v + } +} + +func (b *Batch) TotalDocSize() int { + var s int + for k, v := range b.IndexOps { + if v != nil { + s += v.Size() + sizeOfString + } + s += len(k) + } + return s +} diff --git a/vendor/github.com/blevesearch/bleve_index_api/directory.go b/vendor/github.com/blevesearch/bleve_index_api/directory.go new file mode 100644 index 0000000000..709a384565 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve_index_api/directory.go @@ -0,0 +1,23 @@ +// Copyright (c) 2021 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package index + +import ( + "io" +) + +type Directory interface { + GetWriter(filePath string) (io.WriteCloser, error) +} diff --git a/vendor/github.com/blevesearch/bleve_index_api/document.go b/vendor/github.com/blevesearch/bleve_index_api/document.go new file mode 100644 index 0000000000..a6d0527dfa --- /dev/null +++ b/vendor/github.com/blevesearch/bleve_index_api/document.go @@ -0,0 +1,78 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package index + +import "time" + +type Document interface { + ID() string + Size() int + + VisitFields(visitor FieldVisitor) + VisitComposite(visitor CompositeFieldVisitor) + HasComposite() bool + + NumPlainTextBytes() uint64 + + AddIDField() +} + +type FieldVisitor func(Field) + +type Field interface { + Name() string + Value() []byte + ArrayPositions() []uint64 + + EncodedFieldType() byte + + Analyze() + + Options() FieldIndexingOptions + + AnalyzedLength() int + AnalyzedTokenFrequencies() TokenFrequencies + + NumPlainTextBytes() uint64 +} + +type CompositeFieldVisitor func(field CompositeField) + +type CompositeField interface { + Field + + Compose(field string, length int, freq TokenFrequencies) +} + +type TextField interface { + Text() string +} + +type NumericField interface { + Number() (float64, error) +} + +type DateTimeField interface { + DateTime() (time.Time, error) +} + +type BooleanField interface { + Boolean() (bool, error) +} + +type GeoPointField interface { + Lon() (float64, error) + Lat() (float64, error) +} diff --git a/vendor/github.com/blevesearch/bleve/analysis/freq.go b/vendor/github.com/blevesearch/bleve_index_api/freq.go similarity index 66% rename from vendor/github.com/blevesearch/bleve/analysis/freq.go rename to vendor/github.com/blevesearch/bleve_index_api/freq.go index 198c149b2b..5b6c7e1d82 100644 --- a/vendor/github.com/blevesearch/bleve/analysis/freq.go +++ b/vendor/github.com/blevesearch/bleve_index_api/freq.go @@ -1,4 +1,4 @@ -// Copyright (c) 2014 Couchbase, Inc. +// Copyright (c) 2020 Couchbase, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,13 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -package analysis +package index -import ( - "reflect" - - "github.com/blevesearch/bleve/size" -) +import "reflect" var reflectStaticSizeTokenLocation int var reflectStaticSizeTokenFreq int @@ -44,7 +40,7 @@ type TokenLocation struct { func (tl *TokenLocation) Size() int { rv := reflectStaticSizeTokenLocation - rv += len(tl.ArrayPositions) * size.SizeOfUint64 + rv += len(tl.ArrayPositions) * sizeOfUint64 return rv } @@ -69,13 +65,17 @@ func (tf *TokenFreq) Frequency() int { return tf.frequency } +func (tf *TokenFreq) SetFrequency(frequency int) { + tf.frequency = frequency +} + // TokenFrequencies maps document terms to their combined frequencies from all // fields. type TokenFrequencies map[string]*TokenFreq func (tfs TokenFrequencies) Size() int { - rv := size.SizeOfMap - rv += len(tfs) * (size.SizeOfString + size.SizeOfPtr) + rv := sizeOfMap + rv += len(tfs) * (sizeOfString + sizeOfPtr) for k, v := range tfs { rv += len(k) rv += v.Size() @@ -93,7 +93,7 @@ func (tfs TokenFrequencies) MergeAll(remoteField string, other TokenFrequencies) existingTf, exists := tfs[tfk] if exists { existingTf.Locations = append(existingTf.Locations, tf.Locations...) - existingTf.frequency = existingTf.frequency + tf.frequency + existingTf.frequency += tf.frequency } else { tfs[tfk] = &TokenFreq{ Term: tf.Term, @@ -104,49 +104,3 @@ func (tfs TokenFrequencies) MergeAll(remoteField string, other TokenFrequencies) } } } - -func TokenFrequency(tokens TokenStream, arrayPositions []uint64, includeTermVectors bool) TokenFrequencies { - rv := make(map[string]*TokenFreq, len(tokens)) - - if includeTermVectors { - tls := make([]TokenLocation, len(tokens)) - tlNext := 0 - - for _, token := range tokens { - tls[tlNext] = TokenLocation{ - ArrayPositions: arrayPositions, - Start: token.Start, - End: token.End, - Position: token.Position, - } - - curr, ok := rv[string(token.Term)] - if ok { - curr.Locations = append(curr.Locations, &tls[tlNext]) - curr.frequency++ - } else { - rv[string(token.Term)] = &TokenFreq{ - Term: token.Term, - Locations: []*TokenLocation{&tls[tlNext]}, - frequency: 1, - } - } - - tlNext++ - } - } else { - for _, token := range tokens { - curr, exists := rv[string(token.Term)] - if exists { - curr.frequency++ - } else { - rv[string(token.Term)] = &TokenFreq{ - Term: token.Term, - frequency: 1, - } - } - } - } - - return rv -} diff --git a/vendor/github.com/blevesearch/bleve_index_api/go.mod b/vendor/github.com/blevesearch/bleve_index_api/go.mod new file mode 100644 index 0000000000..f129fb52c3 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve_index_api/go.mod @@ -0,0 +1,3 @@ +module github.com/blevesearch/bleve_index_api + +go 1.13 diff --git a/vendor/github.com/willf/bitset/go.sum b/vendor/github.com/blevesearch/bleve_index_api/go.sum similarity index 100% rename from vendor/github.com/willf/bitset/go.sum rename to vendor/github.com/blevesearch/bleve_index_api/go.sum diff --git a/vendor/github.com/blevesearch/bleve/index/index.go b/vendor/github.com/blevesearch/bleve_index_api/index.go similarity index 54% rename from vendor/github.com/blevesearch/bleve/index/index.go rename to vendor/github.com/blevesearch/bleve_index_api/index.go index 551f8de842..1497919703 100644 --- a/vendor/github.com/blevesearch/bleve/index/index.go +++ b/vendor/github.com/blevesearch/bleve_index_api/index.go @@ -16,13 +16,7 @@ package index import ( "bytes" - "encoding/json" - "fmt" "reflect" - - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/size" ) var reflectStaticSizeTermFieldDoc int @@ -35,13 +29,11 @@ func init() { reflectStaticSizeTermFieldVector = int(reflect.TypeOf(tfv).Size()) } -var ErrorUnknownStorageType = fmt.Errorf("unknown storage type") - type Index interface { Open() error Close() error - Update(doc *document.Document) error + Update(doc Document) error Delete(id string) error Batch(batch *Batch) error @@ -52,16 +44,9 @@ type Index interface { // release associated resources. Reader() (IndexReader, error) - Stats() json.Marshaler StatsMap() map[string]interface{} - - Analyze(d *document.Document) *AnalysisResult - - Advanced() (store.KVStore, error) } -type DocumentFieldTermVisitor func(field string, term []byte) - type IndexReader interface { TermFieldReader(term []byte, field string, includeFreq, includeNorm, includeTermVectors bool) (TermFieldReader, error) @@ -77,8 +62,7 @@ type IndexReader interface { FieldDictRange(field string, startTerm []byte, endTerm []byte) (FieldDict, error) FieldDictPrefix(field string, termPrefix []byte) (FieldDict, error) - Document(id string) (*document.Document, error) - DocumentVisitFieldTerms(id IndexInternalID, fields []string, visitor DocumentFieldTermVisitor) error + Document(id string) (Document, error) DocValueReader(fields []string) (DocValueReader, error) @@ -91,24 +75,9 @@ type IndexReader interface { ExternalID(id IndexInternalID) (string, error) InternalID(id string) (IndexInternalID, error) - DumpAll() chan interface{} - DumpDoc(id string) chan interface{} - DumpFields() chan interface{} - Close() error } -// The Regexp interface defines the subset of the regexp.Regexp API -// methods that are used by bleve indexes, allowing callers to pass in -// alternate implementations. -type Regexp interface { - FindStringIndex(s string) (loc []int) - - LiteralPrefix() (prefix string, complete bool) - - String() string -} - type IndexReaderRegexp interface { FieldDictRegexp(field string, regex string) (FieldDict, error) } @@ -117,37 +86,10 @@ type IndexReaderFuzzy interface { FieldDictFuzzy(field string, term string, fuzziness int, prefix string) (FieldDict, error) } -type IndexReaderOnly interface { - FieldDictOnly(field string, onlyTerms [][]byte, includeCount bool) (FieldDict, error) -} - type IndexReaderContains interface { FieldDictContains(field string) (FieldDictContains, error) } -// FieldTerms contains the terms used by a document, keyed by field -type FieldTerms map[string][]string - -// FieldsNotYetCached returns a list of fields not yet cached out of a larger list of fields -func (f FieldTerms) FieldsNotYetCached(fields []string) []string { - rv := make([]string, 0, len(fields)) - for _, field := range fields { - if _, ok := f[field]; !ok { - rv = append(rv, field) - } - } - return rv -} - -// Merge will combine two FieldTerms -// it assumes that the terms lists are complete (thus do not need to be merged) -// field terms from the other list always replace the ones in the receiver -func (f FieldTerms) Merge(other FieldTerms) { - for field, terms := range other { - f[field] = terms - } -} - type TermFieldVector struct { Field string ArrayPositions []uint64 @@ -157,8 +99,8 @@ type TermFieldVector struct { } func (tfv *TermFieldVector) Size() int { - return reflectStaticSizeTermFieldVector + size.SizeOfPtr + - len(tfv.Field) + len(tfv.ArrayPositions)*size.SizeOfUint64 + return reflectStaticSizeTermFieldVector + sizeOfPtr + + len(tfv.Field) + len(tfv.ArrayPositions)*sizeOfUint64 } // IndexInternalID is an opaque document identifier interal to the index impl @@ -181,7 +123,7 @@ type TermFieldDoc struct { } func (tfd *TermFieldDoc) Size() int { - sizeInBytes := reflectStaticSizeTermFieldDoc + size.SizeOfPtr + + sizeInBytes := reflectStaticSizeTermFieldDoc + sizeOfPtr + len(tfd.Term) + len(tfd.ID) for _, entry := range tfd.Vectors { @@ -256,121 +198,15 @@ type DocIDReader interface { Close() error } -type BatchCallback func(error) - -type Batch struct { - IndexOps map[string]*document.Document - InternalOps map[string][]byte - persistedCallback BatchCallback -} - -func NewBatch() *Batch { - return &Batch{ - IndexOps: make(map[string]*document.Document), - InternalOps: make(map[string][]byte), - } -} - -func (b *Batch) Update(doc *document.Document) { - b.IndexOps[doc.ID] = doc -} - -func (b *Batch) Delete(id string) { - b.IndexOps[id] = nil -} - -func (b *Batch) SetInternal(key, val []byte) { - b.InternalOps[string(key)] = val -} - -func (b *Batch) DeleteInternal(key []byte) { - b.InternalOps[string(key)] = nil -} - -func (b *Batch) SetPersistedCallback(f BatchCallback) { - b.persistedCallback = f -} - -func (b *Batch) PersistedCallback() BatchCallback { - return b.persistedCallback -} - -func (b *Batch) String() string { - rv := fmt.Sprintf("Batch (%d ops, %d internal ops)\n", len(b.IndexOps), len(b.InternalOps)) - for k, v := range b.IndexOps { - if v != nil { - rv += fmt.Sprintf("\tINDEX - '%s'\n", k) - } else { - rv += fmt.Sprintf("\tDELETE - '%s'\n", k) - } - } - for k, v := range b.InternalOps { - if v != nil { - rv += fmt.Sprintf("\tSET INTERNAL - '%s'\n", k) - } else { - rv += fmt.Sprintf("\tDELETE INTERNAL - '%s'\n", k) - } - } - return rv -} - -func (b *Batch) Reset() { - b.IndexOps = make(map[string]*document.Document) - b.InternalOps = make(map[string][]byte) - b.persistedCallback = nil -} - -func (b *Batch) Merge(o *Batch) { - for k, v := range o.IndexOps { - b.IndexOps[k] = v - } - for k, v := range o.InternalOps { - b.InternalOps[k] = v - } -} - -func (b *Batch) TotalDocSize() int { - var s int - for k, v := range b.IndexOps { - if v != nil { - s += v.Size() + size.SizeOfString - } - s += len(k) - } - return s -} - -// Optimizable represents an optional interface that implementable by -// optimizable resources (e.g., TermFieldReaders, Searchers). These -// optimizable resources are provided the same OptimizableContext -// instance, so that they can coordinate via dynamic interface -// casting. -type Optimizable interface { - Optimize(kind string, octx OptimizableContext) (OptimizableContext, error) -} - -// Represents a result of optimization -- see the Finish() method. -type Optimized interface{} - -type OptimizableContext interface { - // Once all the optimzable resources have been provided the same - // OptimizableContext instance, the optimization preparations are - // finished or completed via the Finish() method. - // - // Depending on the optimization being performed, the Finish() - // method might return a non-nil Optimized instance. For example, - // the Optimized instance might represent an optimized - // TermFieldReader instance. - Finish() (Optimized, error) -} +type DocValueVisitor func(field string, term []byte) type DocValueReader interface { - VisitDocValues(id IndexInternalID, visitor DocumentFieldTermVisitor) error + VisitDocValues(id IndexInternalID, visitor DocValueVisitor) error } // IndexBuilder is an interface supported by some index schemes // to allow direct write-only index building type IndexBuilder interface { - Index(doc *document.Document) error + Index(doc Document) error Close() error } diff --git a/vendor/github.com/blevesearch/bleve/document/indexing_options.go b/vendor/github.com/blevesearch/bleve_index_api/indexing_options.go similarity index 67% rename from vendor/github.com/blevesearch/bleve/document/indexing_options.go rename to vendor/github.com/blevesearch/bleve_index_api/indexing_options.go index 44498a8e90..9724ccae07 100644 --- a/vendor/github.com/blevesearch/bleve/document/indexing_options.go +++ b/vendor/github.com/blevesearch/bleve_index_api/indexing_options.go @@ -12,34 +12,39 @@ // See the License for the specific language governing permissions and // limitations under the License. -package document +package index -type IndexingOptions int +type FieldIndexingOptions int const ( - IndexField IndexingOptions = 1 << iota + IndexField FieldIndexingOptions = 1 << iota StoreField IncludeTermVectors DocValues + SkipFreqNorm ) -func (o IndexingOptions) IsIndexed() bool { +func (o FieldIndexingOptions) IsIndexed() bool { return o&IndexField != 0 } -func (o IndexingOptions) IsStored() bool { +func (o FieldIndexingOptions) IsStored() bool { return o&StoreField != 0 } -func (o IndexingOptions) IncludeTermVectors() bool { +func (o FieldIndexingOptions) IncludeTermVectors() bool { return o&IncludeTermVectors != 0 } -func (o IndexingOptions) IncludeDocValues() bool { +func (o FieldIndexingOptions) IncludeDocValues() bool { return o&DocValues != 0 } -func (o IndexingOptions) String() string { +func (o FieldIndexingOptions) SkipFreqNorm() bool { + return o&SkipFreqNorm != 0 +} + +func (o FieldIndexingOptions) String() string { rv := "" if o.IsIndexed() { rv += "INDEXED" @@ -62,5 +67,11 @@ func (o IndexingOptions) String() string { } rv += "DV" } + if !o.SkipFreqNorm() { + if rv != "" { + rv += ", " + } + rv += "FN" + } return rv } diff --git a/vendor/github.com/blevesearch/bleve_index_api/optimize.go b/vendor/github.com/blevesearch/bleve_index_api/optimize.go new file mode 100644 index 0000000000..2b4e1244c4 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve_index_api/optimize.go @@ -0,0 +1,39 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package index + +// Optimizable represents an optional interface that implementable by +// optimizable resources (e.g., TermFieldReaders, Searchers). These +// optimizable resources are provided the same OptimizableContext +// instance, so that they can coordinate via dynamic interface +// casting. +type Optimizable interface { + Optimize(kind string, octx OptimizableContext) (OptimizableContext, error) +} + +// Represents a result of optimization -- see the Finish() method. +type Optimized interface{} + +type OptimizableContext interface { + // Once all the optimzable resources have been provided the same + // OptimizableContext instance, the optimization preparations are + // finished or completed via the Finish() method. + // + // Depending on the optimization being performed, the Finish() + // method might return a non-nil Optimized instance. For example, + // the Optimized instance might represent an optimized + // TermFieldReader instance. + Finish() (Optimized, error) +} diff --git a/vendor/github.com/blevesearch/bleve_index_api/sizes.go b/vendor/github.com/blevesearch/bleve_index_api/sizes.go new file mode 100644 index 0000000000..65c2dd0000 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve_index_api/sizes.go @@ -0,0 +1,35 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package index + +import ( + "reflect" +) + +func init() { + var m map[int]int + sizeOfMap = int(reflect.TypeOf(m).Size()) + var ptr *int + sizeOfPtr = int(reflect.TypeOf(ptr).Size()) + var str string + sizeOfString = int(reflect.TypeOf(str).Size()) + var u64 uint64 + sizeOfUint64 = int(reflect.TypeOf(u64).Size()) +} + +var sizeOfMap int +var sizeOfPtr int +var sizeOfString int +var sizeOfUint64 int diff --git a/vendor/github.com/blevesearch/scorch_segment_api/v2/.golangci.yml b/vendor/github.com/blevesearch/scorch_segment_api/v2/.golangci.yml new file mode 100644 index 0000000000..664f35f27e --- /dev/null +++ b/vendor/github.com/blevesearch/scorch_segment_api/v2/.golangci.yml @@ -0,0 +1,42 @@ +linters: + # please, do not use `enable-all`: it's deprecated and will be removed soon. + # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint + disable-all: true + enable: + - bodyclose + - deadcode + - depguard + - dogsled + - dupl + - errcheck + - funlen + - gochecknoinits + - goconst + - gocritic + - gocyclo + - gofmt + - goimports + - golint + - gomnd + - goprintffuncname + - gosec + - gosimple + - govet + - ineffassign + - interfacer + - lll + - misspell + - nakedret + - nolintlint + - rowserrcheck + - scopelint + - staticcheck + - structcheck + - stylecheck + - typecheck + - unconvert + - unparam + - unused + - varcheck + - whitespace + diff --git a/vendor/github.com/blevesearch/zap/v12/LICENSE b/vendor/github.com/blevesearch/scorch_segment_api/v2/LICENSE similarity index 100% rename from vendor/github.com/blevesearch/zap/v12/LICENSE rename to vendor/github.com/blevesearch/scorch_segment_api/v2/LICENSE diff --git a/vendor/github.com/blevesearch/scorch_segment_api/v2/README.md b/vendor/github.com/blevesearch/scorch_segment_api/v2/README.md new file mode 100644 index 0000000000..dc33b004ed --- /dev/null +++ b/vendor/github.com/blevesearch/scorch_segment_api/v2/README.md @@ -0,0 +1,11 @@ +# Scorch Segment API + +[![PkgGoDev](https://pkg.go.dev/badge/github.com/blevesearch/scorch_segment_api)](https://pkg.go.dev/github.com/blevesearch/scorch_segment_api) +[![Tests](https://github.com/blevesearch/scorch_segment_api/workflows/Tests/badge.svg?branch=master&event=push)](https://github.com/blevesearch/scorch_segment_api/actions?query=workflow%3ATests+event%3Apush+branch%3Amaster) +[![Lint](https://github.com/blevesearch/scorch_segment_api/workflows/Lint/badge.svg?branch=master&event=push)](https://github.com/blevesearch/scorch_segment_api/actions?query=workflow%3ALint+event%3Apush+branch%3Amaster) + +Scorch supports a pluggable Segment interface. + +By placing these interfaces in their own, *hopefully* slowly evolving module, it frees up Scorch and the underlying segment to each introduce new major versions without interfering with one another. + +With that in mind, we anticipate introducing non-breaking changes only to this module, and keeping the major version at 1.x for some time. diff --git a/vendor/github.com/blevesearch/scorch_segment_api/v2/automaton.go b/vendor/github.com/blevesearch/scorch_segment_api/v2/automaton.go new file mode 100644 index 0000000000..4577ceb2e6 --- /dev/null +++ b/vendor/github.com/blevesearch/scorch_segment_api/v2/automaton.go @@ -0,0 +1,36 @@ +// Copyright (c) 2021 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package segment + +// Automaton represents the general contract of a byte-based finite automaton +type Automaton interface { + + // Start returns the start state + Start() int + + // IsMatch returns true if and only if the state is a match + IsMatch(int) bool + + // CanMatch returns true if and only if it is possible to reach a match + // in zero or more steps + CanMatch(int) bool + + // WillAlwaysMatch returns true if and only if the current state matches + // and will always match no matter what steps are taken + WillAlwaysMatch(int) bool + + // Accept returns the next state given the input to the specified state + Accept(int, byte) int +} diff --git a/vendor/github.com/blevesearch/scorch_segment_api/v2/go.mod b/vendor/github.com/blevesearch/scorch_segment_api/v2/go.mod new file mode 100644 index 0000000000..fb60b845ef --- /dev/null +++ b/vendor/github.com/blevesearch/scorch_segment_api/v2/go.mod @@ -0,0 +1,10 @@ +module github.com/blevesearch/scorch_segment_api/v2 + +go 1.13 + +require ( + github.com/RoaringBitmap/roaring v0.4.23 + github.com/blevesearch/bleve_index_api v1.0.0 + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/mschoch/smat v0.2.0 // indirect +) diff --git a/vendor/github.com/blevesearch/scorch_segment_api/v2/go.sum b/vendor/github.com/blevesearch/scorch_segment_api/v2/go.sum new file mode 100644 index 0000000000..c465e59d69 --- /dev/null +++ b/vendor/github.com/blevesearch/scorch_segment_api/v2/go.sum @@ -0,0 +1,35 @@ +github.com/RoaringBitmap/roaring v0.4.23 h1:gpyfd12QohbqhFO4NVDUdoPOCXsyahYRQhINmlHxKeo= +github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= +github.com/blevesearch/bleve_index_api v1.0.0 h1:Ds3XeuTxjXCkG6pgIwWDRyooJKNIuOKemnN0N0IkhTU= +github.com/blevesearch/bleve_index_api v1.0.0/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2 h1:Ujru1hufTHVb++eG6OuNDKMxZnGIvF6o/u8q/8h2+I4= +github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= +github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 h1:gclg6gY70GLy3PbkQ1AERPfmLMMagS60DKF78eWwLn8= +github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99 h1:twflg0XRTjwKpxb/jFExr4HGq6on2dEOmnL6FV+fgPw= +github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= +github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= +github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= +github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ= +github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU= +github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc= +github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/segment/segment.go b/vendor/github.com/blevesearch/scorch_segment_api/v2/segment.go similarity index 78% rename from vendor/github.com/blevesearch/bleve/index/scorch/segment/segment.go rename to vendor/github.com/blevesearch/scorch_segment_api/v2/segment.go index ddd0d09102..702214a55e 100644 --- a/vendor/github.com/blevesearch/bleve/index/scorch/segment/segment.go +++ b/vendor/github.com/blevesearch/scorch_segment_api/v2/segment.go @@ -18,21 +18,20 @@ import ( "fmt" "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index" - "github.com/couchbase/vellum" + index "github.com/blevesearch/bleve_index_api" ) var ErrClosed = fmt.Errorf("index closed") -// DocumentFieldValueVisitor defines a callback to be visited for each +// StoredFieldValueVisitor defines a callback to be visited for each // stored field value. The return value determines if the visitor // should keep going. Returning true continues visiting, false stops. -type DocumentFieldValueVisitor func(field string, typ byte, value []byte, pos []uint64) bool +type StoredFieldValueVisitor func(field string, typ byte, value []byte, pos []uint64) bool type Segment interface { Dictionary(field string) (TermDictionary, error) - VisitDocument(num uint64, visitor DocumentFieldValueVisitor) error + VisitStoredFields(num uint64, visitor StoredFieldValueVisitor) error DocID(num uint64) ([]byte, error) @@ -63,12 +62,8 @@ type PersistedSegment interface { type TermDictionary interface { PostingsList(term []byte, except *roaring.Bitmap, prealloc PostingsList) (PostingsList, error) - Iterator() DictionaryIterator - PrefixIterator(prefix string) DictionaryIterator - RangeIterator(start, end string) DictionaryIterator - AutomatonIterator(a vellum.Automaton, + AutomatonIterator(a Automaton, startKeyInclusive, endKeyExclusive []byte) DictionaryIterator - OnlyIterator(onlyTerms [][]byte, includeCount bool) DictionaryIterator Contains(key []byte) (bool, error) } @@ -132,16 +127,16 @@ type Location interface { Size() int } -// DocumentFieldTermVisitable is implemented by various scorch segment +// DocValueVisitable is implemented by various scorch segment // implementations with persistence for the un inverting of the // postings or other indexed values. -type DocumentFieldTermVisitable interface { - VisitDocumentFieldTerms(localDocNum uint64, fields []string, - visitor index.DocumentFieldTermVisitor, optional DocVisitState) (DocVisitState, error) +type DocValueVisitable interface { + VisitDocValues(localDocNum uint64, fields []string, + visitor index.DocValueVisitor, optional DocVisitState) (DocVisitState, error) // VisitableDocValueFields implementation should return // the list of fields which are document value persisted and - // therefore visitable by the above VisitDocumentFieldTerms method. + // therefore visitable by the above VisitDocValues method. VisitableDocValueFields() ([]string, error) } diff --git a/vendor/github.com/blevesearch/zap/v13/LICENSE b/vendor/github.com/blevesearch/upsidedown_store_api/LICENSE similarity index 100% rename from vendor/github.com/blevesearch/zap/v13/LICENSE rename to vendor/github.com/blevesearch/upsidedown_store_api/LICENSE diff --git a/vendor/github.com/blevesearch/upsidedown_store_api/README.md b/vendor/github.com/blevesearch/upsidedown_store_api/README.md new file mode 100644 index 0000000000..9192819473 --- /dev/null +++ b/vendor/github.com/blevesearch/upsidedown_store_api/README.md @@ -0,0 +1,7 @@ +# Upsidedown Store API + +Upsidedown supports a pluggable Key/Value storage interface. + +By placing these interfaces in their own, *hopefully* slowly evolving module, it frees up Upsidedown and the underlying storage implementations to each introduce new major versions without interfering with one another. + +With that in mind, we anticipate introducing non-breaking changes only to this module, and keeping the major version at 1.x for some time. diff --git a/vendor/github.com/blevesearch/bleve/index/store/batch.go b/vendor/github.com/blevesearch/upsidedown_store_api/batch.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/index/store/batch.go rename to vendor/github.com/blevesearch/upsidedown_store_api/batch.go diff --git a/vendor/github.com/blevesearch/upsidedown_store_api/go.mod b/vendor/github.com/blevesearch/upsidedown_store_api/go.mod new file mode 100644 index 0000000000..1e89f18299 --- /dev/null +++ b/vendor/github.com/blevesearch/upsidedown_store_api/go.mod @@ -0,0 +1,3 @@ +module github.com/blevesearch/upsidedown_store_api + +go 1.13 diff --git a/vendor/github.com/blevesearch/bleve/index/store/kvstore.go b/vendor/github.com/blevesearch/upsidedown_store_api/kvstore.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/index/store/kvstore.go rename to vendor/github.com/blevesearch/upsidedown_store_api/kvstore.go diff --git a/vendor/github.com/blevesearch/bleve/index/store/merge.go b/vendor/github.com/blevesearch/upsidedown_store_api/merge.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/index/store/merge.go rename to vendor/github.com/blevesearch/upsidedown_store_api/merge.go diff --git a/vendor/github.com/blevesearch/bleve/index/store/multiget.go b/vendor/github.com/blevesearch/upsidedown_store_api/multiget.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/index/store/multiget.go rename to vendor/github.com/blevesearch/upsidedown_store_api/multiget.go diff --git a/vendor/github.com/couchbase/vellum/CONTRIBUTING.md b/vendor/github.com/blevesearch/vellum/CONTRIBUTING.md similarity index 100% rename from vendor/github.com/couchbase/vellum/CONTRIBUTING.md rename to vendor/github.com/blevesearch/vellum/CONTRIBUTING.md diff --git a/vendor/github.com/blevesearch/zap/v14/LICENSE b/vendor/github.com/blevesearch/vellum/LICENSE similarity index 100% rename from vendor/github.com/blevesearch/zap/v14/LICENSE rename to vendor/github.com/blevesearch/vellum/LICENSE diff --git a/vendor/github.com/couchbase/vellum/README.md b/vendor/github.com/blevesearch/vellum/README.md similarity index 100% rename from vendor/github.com/couchbase/vellum/README.md rename to vendor/github.com/blevesearch/vellum/README.md diff --git a/vendor/github.com/couchbase/vellum/automaton.go b/vendor/github.com/blevesearch/vellum/automaton.go similarity index 100% rename from vendor/github.com/couchbase/vellum/automaton.go rename to vendor/github.com/blevesearch/vellum/automaton.go diff --git a/vendor/github.com/couchbase/vellum/builder.go b/vendor/github.com/blevesearch/vellum/builder.go similarity index 100% rename from vendor/github.com/couchbase/vellum/builder.go rename to vendor/github.com/blevesearch/vellum/builder.go diff --git a/vendor/github.com/couchbase/vellum/common.go b/vendor/github.com/blevesearch/vellum/common.go similarity index 100% rename from vendor/github.com/couchbase/vellum/common.go rename to vendor/github.com/blevesearch/vellum/common.go diff --git a/vendor/github.com/couchbase/vellum/decoder_v1.go b/vendor/github.com/blevesearch/vellum/decoder_v1.go similarity index 100% rename from vendor/github.com/couchbase/vellum/decoder_v1.go rename to vendor/github.com/blevesearch/vellum/decoder_v1.go diff --git a/vendor/github.com/couchbase/vellum/encoder_v1.go b/vendor/github.com/blevesearch/vellum/encoder_v1.go similarity index 100% rename from vendor/github.com/couchbase/vellum/encoder_v1.go rename to vendor/github.com/blevesearch/vellum/encoder_v1.go diff --git a/vendor/github.com/couchbase/vellum/encoding.go b/vendor/github.com/blevesearch/vellum/encoding.go similarity index 100% rename from vendor/github.com/couchbase/vellum/encoding.go rename to vendor/github.com/blevesearch/vellum/encoding.go diff --git a/vendor/github.com/couchbase/vellum/fst.go b/vendor/github.com/blevesearch/vellum/fst.go similarity index 99% rename from vendor/github.com/couchbase/vellum/fst.go rename to vendor/github.com/blevesearch/vellum/fst.go index 64ee21a410..3140042b64 100644 --- a/vendor/github.com/couchbase/vellum/fst.go +++ b/vendor/github.com/blevesearch/vellum/fst.go @@ -17,7 +17,7 @@ package vellum import ( "io" - "github.com/willf/bitset" + "github.com/bits-and-blooms/bitset" ) // FST is an in-memory representation of a finite state transducer, diff --git a/vendor/github.com/couchbase/vellum/fst_iterator.go b/vendor/github.com/blevesearch/vellum/fst_iterator.go similarity index 100% rename from vendor/github.com/couchbase/vellum/fst_iterator.go rename to vendor/github.com/blevesearch/vellum/fst_iterator.go diff --git a/vendor/github.com/couchbase/vellum/go.mod b/vendor/github.com/blevesearch/vellum/go.mod similarity index 66% rename from vendor/github.com/couchbase/vellum/go.mod rename to vendor/github.com/blevesearch/vellum/go.mod index ef9d9abfb0..4678aa8fa8 100644 --- a/vendor/github.com/couchbase/vellum/go.mod +++ b/vendor/github.com/blevesearch/vellum/go.mod @@ -1,10 +1,10 @@ -module github.com/couchbase/vellum +module github.com/blevesearch/vellum go 1.12 require ( + github.com/bits-and-blooms/bitset v1.2.0 github.com/blevesearch/mmap-go v1.0.2 github.com/spf13/cobra v0.0.5 - github.com/willf/bitset v1.1.10 golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a // indirect ) diff --git a/vendor/github.com/couchbase/vellum/go.sum b/vendor/github.com/blevesearch/vellum/go.sum similarity index 95% rename from vendor/github.com/couchbase/vellum/go.sum rename to vendor/github.com/blevesearch/vellum/go.sum index 1022e684eb..676efb69c1 100644 --- a/vendor/github.com/couchbase/vellum/go.sum +++ b/vendor/github.com/blevesearch/vellum/go.sum @@ -1,5 +1,7 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= +github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/blevesearch/mmap-go v1.0.2 h1:JtMHb+FgQCTTYIhtMvimw15dJwu1Y5lrZDMOFXVWPk0= github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -27,8 +29,6 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc= -github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/vendor/github.com/couchbase/vellum/levenshtein/LICENSE b/vendor/github.com/blevesearch/vellum/levenshtein/LICENSE similarity index 100% rename from vendor/github.com/couchbase/vellum/levenshtein/LICENSE rename to vendor/github.com/blevesearch/vellum/levenshtein/LICENSE diff --git a/vendor/github.com/couchbase/vellum/levenshtein/README.md b/vendor/github.com/blevesearch/vellum/levenshtein/README.md similarity index 100% rename from vendor/github.com/couchbase/vellum/levenshtein/README.md rename to vendor/github.com/blevesearch/vellum/levenshtein/README.md diff --git a/vendor/github.com/couchbase/vellum/levenshtein/alphabet.go b/vendor/github.com/blevesearch/vellum/levenshtein/alphabet.go similarity index 100% rename from vendor/github.com/couchbase/vellum/levenshtein/alphabet.go rename to vendor/github.com/blevesearch/vellum/levenshtein/alphabet.go diff --git a/vendor/github.com/couchbase/vellum/levenshtein/dfa.go b/vendor/github.com/blevesearch/vellum/levenshtein/dfa.go similarity index 100% rename from vendor/github.com/couchbase/vellum/levenshtein/dfa.go rename to vendor/github.com/blevesearch/vellum/levenshtein/dfa.go diff --git a/vendor/github.com/couchbase/vellum/levenshtein/levenshtein.go b/vendor/github.com/blevesearch/vellum/levenshtein/levenshtein.go similarity index 100% rename from vendor/github.com/couchbase/vellum/levenshtein/levenshtein.go rename to vendor/github.com/blevesearch/vellum/levenshtein/levenshtein.go diff --git a/vendor/github.com/couchbase/vellum/levenshtein/levenshtein_nfa.go b/vendor/github.com/blevesearch/vellum/levenshtein/levenshtein_nfa.go similarity index 100% rename from vendor/github.com/couchbase/vellum/levenshtein/levenshtein_nfa.go rename to vendor/github.com/blevesearch/vellum/levenshtein/levenshtein_nfa.go diff --git a/vendor/github.com/couchbase/vellum/levenshtein/parametric_dfa.go b/vendor/github.com/blevesearch/vellum/levenshtein/parametric_dfa.go similarity index 100% rename from vendor/github.com/couchbase/vellum/levenshtein/parametric_dfa.go rename to vendor/github.com/blevesearch/vellum/levenshtein/parametric_dfa.go diff --git a/vendor/github.com/couchbase/vellum/merge_iterator.go b/vendor/github.com/blevesearch/vellum/merge_iterator.go similarity index 100% rename from vendor/github.com/couchbase/vellum/merge_iterator.go rename to vendor/github.com/blevesearch/vellum/merge_iterator.go diff --git a/vendor/github.com/couchbase/vellum/pack.go b/vendor/github.com/blevesearch/vellum/pack.go similarity index 100% rename from vendor/github.com/couchbase/vellum/pack.go rename to vendor/github.com/blevesearch/vellum/pack.go diff --git a/vendor/github.com/couchbase/vellum/regexp/compile.go b/vendor/github.com/blevesearch/vellum/regexp/compile.go similarity index 99% rename from vendor/github.com/couchbase/vellum/regexp/compile.go rename to vendor/github.com/blevesearch/vellum/regexp/compile.go index 92284d0a87..558ce41985 100644 --- a/vendor/github.com/couchbase/vellum/regexp/compile.go +++ b/vendor/github.com/blevesearch/vellum/regexp/compile.go @@ -20,7 +20,7 @@ import ( unicode_utf8 "unicode/utf8" - "github.com/couchbase/vellum/utf8" + "github.com/blevesearch/vellum/utf8" ) type compiler struct { diff --git a/vendor/github.com/couchbase/vellum/regexp/dfa.go b/vendor/github.com/blevesearch/vellum/regexp/dfa.go similarity index 100% rename from vendor/github.com/couchbase/vellum/regexp/dfa.go rename to vendor/github.com/blevesearch/vellum/regexp/dfa.go diff --git a/vendor/github.com/couchbase/vellum/regexp/inst.go b/vendor/github.com/blevesearch/vellum/regexp/inst.go similarity index 100% rename from vendor/github.com/couchbase/vellum/regexp/inst.go rename to vendor/github.com/blevesearch/vellum/regexp/inst.go diff --git a/vendor/github.com/couchbase/vellum/regexp/regexp.go b/vendor/github.com/blevesearch/vellum/regexp/regexp.go similarity index 100% rename from vendor/github.com/couchbase/vellum/regexp/regexp.go rename to vendor/github.com/blevesearch/vellum/regexp/regexp.go diff --git a/vendor/github.com/couchbase/vellum/regexp/sparse.go b/vendor/github.com/blevesearch/vellum/regexp/sparse.go similarity index 100% rename from vendor/github.com/couchbase/vellum/regexp/sparse.go rename to vendor/github.com/blevesearch/vellum/regexp/sparse.go diff --git a/vendor/github.com/couchbase/vellum/registry.go b/vendor/github.com/blevesearch/vellum/registry.go similarity index 100% rename from vendor/github.com/couchbase/vellum/registry.go rename to vendor/github.com/blevesearch/vellum/registry.go diff --git a/vendor/github.com/couchbase/vellum/transducer.go b/vendor/github.com/blevesearch/vellum/transducer.go similarity index 100% rename from vendor/github.com/couchbase/vellum/transducer.go rename to vendor/github.com/blevesearch/vellum/transducer.go diff --git a/vendor/github.com/couchbase/vellum/utf8/utf8.go b/vendor/github.com/blevesearch/vellum/utf8/utf8.go similarity index 100% rename from vendor/github.com/couchbase/vellum/utf8/utf8.go rename to vendor/github.com/blevesearch/vellum/utf8/utf8.go diff --git a/vendor/github.com/couchbase/vellum/vellum.go b/vendor/github.com/blevesearch/vellum/vellum.go similarity index 100% rename from vendor/github.com/couchbase/vellum/vellum.go rename to vendor/github.com/blevesearch/vellum/vellum.go diff --git a/vendor/github.com/couchbase/vellum/vellum_mmap.go b/vendor/github.com/blevesearch/vellum/vellum_mmap.go similarity index 100% rename from vendor/github.com/couchbase/vellum/vellum_mmap.go rename to vendor/github.com/blevesearch/vellum/vellum_mmap.go diff --git a/vendor/github.com/couchbase/vellum/vellum_nommap.go b/vendor/github.com/blevesearch/vellum/vellum_nommap.go similarity index 100% rename from vendor/github.com/couchbase/vellum/vellum_nommap.go rename to vendor/github.com/blevesearch/vellum/vellum_nommap.go diff --git a/vendor/github.com/couchbase/vellum/writer.go b/vendor/github.com/blevesearch/vellum/writer.go similarity index 100% rename from vendor/github.com/couchbase/vellum/writer.go rename to vendor/github.com/blevesearch/vellum/writer.go diff --git a/vendor/github.com/blevesearch/zap/v11/go.mod b/vendor/github.com/blevesearch/zap/v11/go.mod deleted file mode 100644 index 653859c3bb..0000000000 --- a/vendor/github.com/blevesearch/zap/v11/go.mod +++ /dev/null @@ -1,12 +0,0 @@ -module github.com/blevesearch/zap/v11 - -go 1.12 - -require ( - github.com/RoaringBitmap/roaring v0.4.23 - github.com/blevesearch/bleve v1.0.14 - github.com/blevesearch/mmap-go v1.0.2 - github.com/couchbase/vellum v1.0.2 - github.com/golang/snappy v0.0.1 - github.com/spf13/cobra v0.0.5 -) diff --git a/vendor/github.com/blevesearch/zap/v12/go.mod b/vendor/github.com/blevesearch/zap/v12/go.mod deleted file mode 100644 index 5ff4d9b1be..0000000000 --- a/vendor/github.com/blevesearch/zap/v12/go.mod +++ /dev/null @@ -1,12 +0,0 @@ -module github.com/blevesearch/zap/v12 - -go 1.12 - -require ( - github.com/RoaringBitmap/roaring v0.4.23 - github.com/blevesearch/bleve v1.0.14 - github.com/blevesearch/mmap-go v1.0.2 - github.com/couchbase/vellum v1.0.2 - github.com/golang/snappy v0.0.1 - github.com/spf13/cobra v0.0.5 -) diff --git a/vendor/github.com/blevesearch/zap/v13/go.mod b/vendor/github.com/blevesearch/zap/v13/go.mod deleted file mode 100644 index 91f819195f..0000000000 --- a/vendor/github.com/blevesearch/zap/v13/go.mod +++ /dev/null @@ -1,12 +0,0 @@ -module github.com/blevesearch/zap/v13 - -go 1.12 - -require ( - github.com/RoaringBitmap/roaring v0.4.23 - github.com/blevesearch/bleve v1.0.14 - github.com/blevesearch/mmap-go v1.0.2 - github.com/couchbase/vellum v1.0.2 - github.com/golang/snappy v0.0.1 - github.com/spf13/cobra v0.0.5 -) diff --git a/vendor/github.com/blevesearch/zap/v14/go.mod b/vendor/github.com/blevesearch/zap/v14/go.mod deleted file mode 100644 index 2f4882a0c9..0000000000 --- a/vendor/github.com/blevesearch/zap/v14/go.mod +++ /dev/null @@ -1,12 +0,0 @@ -module github.com/blevesearch/zap/v14 - -go 1.12 - -require ( - github.com/RoaringBitmap/roaring v0.4.23 - github.com/blevesearch/bleve v1.0.14 - github.com/blevesearch/mmap-go v1.0.2 - github.com/couchbase/vellum v1.0.2 - github.com/golang/snappy v0.0.1 - github.com/spf13/cobra v0.0.5 -) diff --git a/vendor/github.com/blevesearch/zap/v15/README.md b/vendor/github.com/blevesearch/zap/v15/README.md deleted file mode 100644 index 0facb669fd..0000000000 --- a/vendor/github.com/blevesearch/zap/v15/README.md +++ /dev/null @@ -1,158 +0,0 @@ -# zap file format - -Advanced ZAP File Format Documentation is [here](zap.md). - -The file is written in the reverse order that we typically access data. This helps us write in one pass since later sections of the file require file offsets of things we've already written. - -Current usage: - -- mmap the entire file -- crc-32 bytes and version are in fixed position at end of the file -- reading remainder of footer could be version specific -- remainder of footer gives us: - - 3 important offsets (docValue , fields index and stored data index) - - 2 important values (number of docs and chunk factor) -- field data is processed once and memoized onto the heap so that we never have to go back to disk for it -- access to stored data by doc number means first navigating to the stored data index, then accessing a fixed position offset into that slice, which gives us the actual address of the data. the first bytes of that section tell us the size of data so that we know where it ends. -- access to all other indexed data follows the following pattern: - - first know the field name -> convert to id - - next navigate to term dictionary for that field - - some operations stop here and do dictionary ops - - next use dictionary to navigate to posting list for a specific term - - walk posting list - - if necessary, walk posting details as we go - - if location info is desired, consult location bitmap to see if it is there - -## stored fields section - -- for each document - - preparation phase: - - produce a slice of metadata bytes and data bytes - - produce these slices in field id order - - field value is appended to the data slice - - metadata slice is varint encoded with the following values for each field value - - field id (uint16) - - field type (byte) - - field value start offset in uncompressed data slice (uint64) - - field value length (uint64) - - field number of array positions (uint64) - - one additional value for each array position (uint64) - - compress the data slice using snappy - - file writing phase: - - remember the start offset for this document - - write out meta data length (varint uint64) - - write out compressed data length (varint uint64) - - write out the metadata bytes - - write out the compressed data bytes - -## stored fields idx - -- for each document - - write start offset (remembered from previous section) of stored data (big endian uint64) - -With this index and a known document number, we have direct access to all the stored field data. - -## posting details (freq/norm) section - -- for each posting list - - produce a slice containing multiple consecutive chunks (each chunk is varint stream) - - produce a slice remembering offsets of where each chunk starts - - preparation phase: - - for each hit in the posting list - - if this hit is in next chunk close out encoding of last chunk and record offset start of next - - encode term frequency (uint64) - - encode norm factor (float32) - - file writing phase: - - remember start position for this posting list details - - write out number of chunks that follow (varint uint64) - - write out length of each chunk (each a varint uint64) - - write out the byte slice containing all the chunk data - -If you know the doc number you're interested in, this format lets you jump to the correct chunk (docNum/chunkFactor) directly and then seek within that chunk until you find it. - -## posting details (location) section - -- for each posting list - - produce a slice containing multiple consecutive chunks (each chunk is varint stream) - - produce a slice remembering offsets of where each chunk starts - - preparation phase: - - for each hit in the posting list - - if this hit is in next chunk close out encoding of last chunk and record offset start of next - - encode field (uint16) - - encode field pos (uint64) - - encode field start (uint64) - - encode field end (uint64) - - encode number of array positions to follow (uint64) - - encode each array position (each uint64) - - file writing phase: - - remember start position for this posting list details - - write out number of chunks that follow (varint uint64) - - write out length of each chunk (each a varint uint64) - - write out the byte slice containing all the chunk data - -If you know the doc number you're interested in, this format lets you jump to the correct chunk (docNum/chunkFactor) directly and then seek within that chunk until you find it. - -## postings list section - -- for each posting list - - preparation phase: - - encode roaring bitmap posting list to bytes (so we know the length) - - file writing phase: - - remember the start position for this posting list - - write freq/norm details offset (remembered from previous, as varint uint64) - - write location details offset (remembered from previous, as varint uint64) - - write length of encoded roaring bitmap - - write the serialized roaring bitmap data - -## dictionary - -- for each field - - preparation phase: - - encode vellum FST with dictionary data pointing to file offset of posting list (remembered from previous) - - file writing phase: - - remember the start position of this persistDictionary - - write length of vellum data (varint uint64) - - write out vellum data - -## fields section - -- for each field - - file writing phase: - - remember start offset for each field - - write dictionary address (remembered from previous) (varint uint64) - - write length of field name (varint uint64) - - write field name bytes - -## fields idx - -- for each field - - file writing phase: - - write big endian uint64 of start offset for each field - -NOTE: currently we don't know or record the length of this fields index. Instead we rely on the fact that we know it immediately precedes a footer of known size. - -## fields DocValue - -- for each field - - preparation phase: - - produce a slice containing multiple consecutive chunks, where each chunk is composed of a meta section followed by compressed columnar field data - - produce a slice remembering the length of each chunk - - file writing phase: - - remember the start position of this first field DocValue offset in the footer - - write out number of chunks that follow (varint uint64) - - write out length of each chunk (each a varint uint64) - - write out the byte slice containing all the chunk data - -NOTE: currently the meta header inside each chunk gives clue to the location offsets and size of the data pertaining to a given docID and any -read operation leverage that meta information to extract the document specific data from the file. - -## footer - -- file writing phase - - write number of docs (big endian uint64) - - write stored field index location (big endian uint64) - - write field index location (big endian uint64) - - write field docValue location (big endian uint64) - - write out chunk factor (big endian uint32) - - write out version (big endian uint32) - - write out file CRC of everything preceding this (big endian uint32) diff --git a/vendor/github.com/blevesearch/zap/v15/count.go b/vendor/github.com/blevesearch/zap/v15/count.go deleted file mode 100644 index 50290f8882..0000000000 --- a/vendor/github.com/blevesearch/zap/v15/count.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "hash/crc32" - "io" - - "github.com/blevesearch/bleve/index/scorch/segment" -) - -// CountHashWriter is a wrapper around a Writer which counts the number of -// bytes which have been written and computes a crc32 hash -type CountHashWriter struct { - w io.Writer - crc uint32 - n int - s segment.StatsReporter -} - -// NewCountHashWriter returns a CountHashWriter which wraps the provided Writer -func NewCountHashWriter(w io.Writer) *CountHashWriter { - return &CountHashWriter{w: w} -} - -func NewCountHashWriterWithStatsReporter(w io.Writer, s segment.StatsReporter) *CountHashWriter { - return &CountHashWriter{w: w, s: s} -} - -// Write writes the provided bytes to the wrapped writer and counts the bytes -func (c *CountHashWriter) Write(b []byte) (int, error) { - n, err := c.w.Write(b) - c.crc = crc32.Update(c.crc, crc32.IEEETable, b[:n]) - c.n += n - if c.s != nil { - c.s.ReportBytesWritten(uint64(n)) - } - return n, err -} - -// Count returns the number of bytes written -func (c *CountHashWriter) Count() int { - return c.n -} - -// Sum32 returns the CRC-32 hash of the content written to this writer -func (c *CountHashWriter) Sum32() uint32 { - return c.crc -} diff --git a/vendor/github.com/blevesearch/zap/v15/dict.go b/vendor/github.com/blevesearch/zap/v15/dict.go deleted file mode 100644 index ad4a8f8dc5..0000000000 --- a/vendor/github.com/blevesearch/zap/v15/dict.go +++ /dev/null @@ -1,263 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bytes" - "fmt" - - "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/couchbase/vellum" -) - -// Dictionary is the zap representation of the term dictionary -type Dictionary struct { - sb *SegmentBase - field string - fieldID uint16 - fst *vellum.FST - fstReader *vellum.Reader -} - -// PostingsList returns the postings list for the specified term -func (d *Dictionary) PostingsList(term []byte, except *roaring.Bitmap, - prealloc segment.PostingsList) (segment.PostingsList, error) { - var preallocPL *PostingsList - pl, ok := prealloc.(*PostingsList) - if ok && pl != nil { - preallocPL = pl - } - return d.postingsList(term, except, preallocPL) -} - -func (d *Dictionary) postingsList(term []byte, except *roaring.Bitmap, rv *PostingsList) (*PostingsList, error) { - if d.fstReader == nil { - if rv == nil || rv == emptyPostingsList { - return emptyPostingsList, nil - } - return d.postingsListInit(rv, except), nil - } - - postingsOffset, exists, err := d.fstReader.Get(term) - if err != nil { - return nil, fmt.Errorf("vellum err: %v", err) - } - if !exists { - if rv == nil || rv == emptyPostingsList { - return emptyPostingsList, nil - } - return d.postingsListInit(rv, except), nil - } - - return d.postingsListFromOffset(postingsOffset, except, rv) -} - -func (d *Dictionary) postingsListFromOffset(postingsOffset uint64, except *roaring.Bitmap, rv *PostingsList) (*PostingsList, error) { - rv = d.postingsListInit(rv, except) - - err := rv.read(postingsOffset, d) - if err != nil { - return nil, err - } - - return rv, nil -} - -func (d *Dictionary) postingsListInit(rv *PostingsList, except *roaring.Bitmap) *PostingsList { - if rv == nil || rv == emptyPostingsList { - rv = &PostingsList{} - } else { - postings := rv.postings - if postings != nil { - postings.Clear() - } - - *rv = PostingsList{} // clear the struct - - rv.postings = postings - } - rv.sb = d.sb - rv.except = except - return rv -} - -func (d *Dictionary) Contains(key []byte) (bool, error) { - return d.fst.Contains(key) -} - -// Iterator returns an iterator for this dictionary -func (d *Dictionary) Iterator() segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - - if d.fst != nil { - itr, err := d.fst.Iterator(nil, nil) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - } - - return rv -} - -// PrefixIterator returns an iterator which only visits terms having the -// the specified prefix -func (d *Dictionary) PrefixIterator(prefix string) segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - - kBeg := []byte(prefix) - kEnd := segment.IncrementBytes(kBeg) - - if d.fst != nil { - itr, err := d.fst.Iterator(kBeg, kEnd) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - } - - return rv -} - -// RangeIterator returns an iterator which only visits terms between the -// start and end terms. NOTE: bleve.index API specifies the end is inclusive. -func (d *Dictionary) RangeIterator(start, end string) segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - - // need to increment the end position to be inclusive - var endBytes []byte - if len(end) > 0 { - endBytes = []byte(end) - if endBytes[len(endBytes)-1] < 0xff { - endBytes[len(endBytes)-1]++ - } else { - endBytes = append(endBytes, 0xff) - } - } - - if d.fst != nil { - itr, err := d.fst.Iterator([]byte(start), endBytes) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - } - - return rv -} - -// AutomatonIterator returns an iterator which only visits terms -// having the the vellum automaton and start/end key range -func (d *Dictionary) AutomatonIterator(a vellum.Automaton, - startKeyInclusive, endKeyExclusive []byte) segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - - if d.fst != nil { - itr, err := d.fst.Search(a, startKeyInclusive, endKeyExclusive) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - } - - return rv -} - -func (d *Dictionary) OnlyIterator(onlyTerms [][]byte, - includeCount bool) segment.DictionaryIterator { - - rv := &DictionaryIterator{ - d: d, - omitCount: !includeCount, - } - - var buf bytes.Buffer - builder, err := vellum.New(&buf, nil) - if err != nil { - rv.err = err - return rv - } - for _, term := range onlyTerms { - err = builder.Insert(term, 0) - if err != nil { - rv.err = err - return rv - } - } - err = builder.Close() - if err != nil { - rv.err = err - return rv - } - - onlyFST, err := vellum.Load(buf.Bytes()) - if err != nil { - rv.err = err - return rv - } - - itr, err := d.fst.Search(onlyFST, nil, nil) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - - return rv -} - -// DictionaryIterator is an iterator for term dictionary -type DictionaryIterator struct { - d *Dictionary - itr vellum.Iterator - err error - tmp PostingsList - entry index.DictEntry - omitCount bool -} - -// Next returns the next entry in the dictionary -func (i *DictionaryIterator) Next() (*index.DictEntry, error) { - if i.err != nil && i.err != vellum.ErrIteratorDone { - return nil, i.err - } else if i.itr == nil || i.err == vellum.ErrIteratorDone { - return nil, nil - } - term, postingsOffset := i.itr.Current() - i.entry.Term = string(term) - if !i.omitCount { - i.err = i.tmp.read(postingsOffset, i.d) - if i.err != nil { - return nil, i.err - } - i.entry.Count = i.tmp.Count() - } - i.err = i.itr.Next() - return &i.entry, nil -} diff --git a/vendor/github.com/blevesearch/zap/v15/go.mod b/vendor/github.com/blevesearch/zap/v15/go.mod deleted file mode 100644 index fcd7d38260..0000000000 --- a/vendor/github.com/blevesearch/zap/v15/go.mod +++ /dev/null @@ -1,12 +0,0 @@ -module github.com/blevesearch/zap/v15 - -go 1.12 - -require ( - github.com/RoaringBitmap/roaring v0.4.23 - github.com/blevesearch/bleve v1.0.14 - github.com/blevesearch/mmap-go v1.0.2 - github.com/couchbase/vellum v1.0.2 - github.com/golang/snappy v0.0.1 - github.com/spf13/cobra v0.0.5 -) diff --git a/vendor/github.com/blevesearch/zap/v15/plugin.go b/vendor/github.com/blevesearch/zap/v15/plugin.go deleted file mode 100644 index 38a0638d4d..0000000000 --- a/vendor/github.com/blevesearch/zap/v15/plugin.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) 2020 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "github.com/blevesearch/bleve/index/scorch/segment" -) - -// ZapPlugin implements the Plugin interface of -// the blevesearch/bleve/index/scorch/segment pkg -type ZapPlugin struct{} - -func (*ZapPlugin) Type() string { - return Type -} - -func (*ZapPlugin) Version() uint32 { - return Version -} - -// Plugin returns an instance segment.Plugin for use -// by the Scorch indexing scheme -func Plugin() segment.Plugin { - return &ZapPlugin{} -} diff --git a/vendor/github.com/blevesearch/zap/v11/.gitignore b/vendor/github.com/blevesearch/zapx/v11/.gitignore similarity index 100% rename from vendor/github.com/blevesearch/zap/v11/.gitignore rename to vendor/github.com/blevesearch/zapx/v11/.gitignore diff --git a/vendor/github.com/blevesearch/zapx/v11/.golangci.yml b/vendor/github.com/blevesearch/zapx/v11/.golangci.yml new file mode 100644 index 0000000000..f0f2f6067f --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v11/.golangci.yml @@ -0,0 +1,29 @@ +linters: + # please, do not use `enable-all`: it's deprecated and will be removed soon. + # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint + disable-all: true + enable: + - bodyclose + - deadcode + - depguard + - dupl + - errcheck + - gofmt + - goimports + - goprintffuncname + - gosec + - gosimple + - govet + - ineffassign + - interfacer + - misspell + - nakedret + - nolintlint + - rowserrcheck + - staticcheck + - structcheck + - typecheck + - unused + - varcheck + - whitespace + diff --git a/vendor/github.com/blevesearch/zap/v15/LICENSE b/vendor/github.com/blevesearch/zapx/v11/LICENSE similarity index 100% rename from vendor/github.com/blevesearch/zap/v15/LICENSE rename to vendor/github.com/blevesearch/zapx/v11/LICENSE diff --git a/vendor/github.com/blevesearch/zap/v14/README.md b/vendor/github.com/blevesearch/zapx/v11/README.md similarity index 94% rename from vendor/github.com/blevesearch/zap/v14/README.md rename to vendor/github.com/blevesearch/zapx/v11/README.md index 0facb669fd..4cbf1a145b 100644 --- a/vendor/github.com/blevesearch/zap/v14/README.md +++ b/vendor/github.com/blevesearch/zapx/v11/README.md @@ -1,4 +1,9 @@ -# zap file format +# zapx file format + +The zapx module is fork of [zap](https://github.com/blevesearch/zap) module which maintains file format compatibility, but removes dependency on bleve, and instead depends only on the indepenent interface modules: + +- [bleve_index_api](https://github.com/blevesearch/scorch_segment_api) +- [scorch_segment_api](https://github.com/blevesearch/scorch_segment_api) Advanced ZAP File Format Documentation is [here](zap.md). diff --git a/vendor/github.com/blevesearch/zap/v11/build.go b/vendor/github.com/blevesearch/zapx/v11/build.go similarity index 79% rename from vendor/github.com/blevesearch/zap/v11/build.go rename to vendor/github.com/blevesearch/zapx/v11/build.go index bac1edb549..3f13a2a6c6 100644 --- a/vendor/github.com/blevesearch/zap/v11/build.go +++ b/vendor/github.com/blevesearch/zapx/v11/build.go @@ -16,10 +16,12 @@ package zap import ( "bufio" + "fmt" + "io" "math" "os" - "github.com/couchbase/vellum" + "github.com/blevesearch/vellum" ) const Version uint32 = 11 @@ -32,6 +34,16 @@ func (sb *SegmentBase) Persist(path string) error { return PersistSegmentBase(sb, path) } +// WriteTo is an implementation of io.WriterTo interface. +func (sb *SegmentBase) WriteTo(w io.Writer) (int64, error) { + if w == nil { + return 0, fmt.Errorf("invalid writer found") + } + + n, err := persistSegmentBaseToWriter(sb, w) + return int64(n), err +} + // PersistSegmentBase persists SegmentBase in the zap file format. func PersistSegmentBase(sb *SegmentBase, path string) error { flag := os.O_RDWR | os.O_CREATE @@ -46,22 +58,7 @@ func PersistSegmentBase(sb *SegmentBase, path string) error { _ = os.Remove(path) } - br := bufio.NewWriter(f) - - _, err = br.Write(sb.mem) - if err != nil { - cleanup() - return err - } - - err = persistFooter(sb.numDocs, sb.storedIndexOffset, sb.fieldsIndexOffset, sb.docValueOffset, - sb.chunkFactor, sb.memCRC, br) - if err != nil { - cleanup() - return err - } - - err = br.Flush() + _, err = persistSegmentBaseToWriter(sb, f) if err != nil { cleanup() return err @@ -79,7 +76,40 @@ func PersistSegmentBase(sb *SegmentBase, path string) error { return err } - return nil + return err +} + +type bufWriter struct { + w *bufio.Writer + n int +} + +func (br *bufWriter) Write(in []byte) (int, error) { + n, err := br.w.Write(in) + br.n += n + return n, err +} + +func persistSegmentBaseToWriter(sb *SegmentBase, w io.Writer) (int, error) { + br := &bufWriter{w: bufio.NewWriter(w)} + + _, err := br.Write(sb.mem) + if err != nil { + return 0, err + } + + err = persistFooter(sb.numDocs, sb.storedIndexOffset, sb.fieldsIndexOffset, + sb.docValueOffset, sb.chunkFactor, sb.memCRC, br) + if err != nil { + return 0, err + } + + err = br.w.Flush() + if err != nil { + return 0, err + } + + return br.n, nil } func persistStoredFieldValues(fieldID int, diff --git a/vendor/github.com/blevesearch/zap/v11/contentcoder.go b/vendor/github.com/blevesearch/zapx/v11/contentcoder.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v11/contentcoder.go rename to vendor/github.com/blevesearch/zapx/v11/contentcoder.go diff --git a/vendor/github.com/blevesearch/zap/v12/count.go b/vendor/github.com/blevesearch/zapx/v11/count.go similarity index 96% rename from vendor/github.com/blevesearch/zap/v12/count.go rename to vendor/github.com/blevesearch/zapx/v11/count.go index 50290f8882..b6135359fb 100644 --- a/vendor/github.com/blevesearch/zap/v12/count.go +++ b/vendor/github.com/blevesearch/zapx/v11/count.go @@ -18,7 +18,7 @@ import ( "hash/crc32" "io" - "github.com/blevesearch/bleve/index/scorch/segment" + segment "github.com/blevesearch/scorch_segment_api/v2" ) // CountHashWriter is a wrapper around a Writer which counts the number of diff --git a/vendor/github.com/blevesearch/zap/v13/dict.go b/vendor/github.com/blevesearch/zapx/v11/dict.go similarity index 60% rename from vendor/github.com/blevesearch/zap/v13/dict.go rename to vendor/github.com/blevesearch/zapx/v11/dict.go index ad4a8f8dc5..e30bf2420d 100644 --- a/vendor/github.com/blevesearch/zap/v13/dict.go +++ b/vendor/github.com/blevesearch/zapx/v11/dict.go @@ -15,13 +15,12 @@ package zap import ( - "bytes" "fmt" "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/couchbase/vellum" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" + "github.com/blevesearch/vellum" ) // Dictionary is the zap representation of the term dictionary @@ -33,6 +32,9 @@ type Dictionary struct { fstReader *vellum.Reader } +// represents an immutable, empty dictionary +var emptyDictionary = &Dictionary{} + // PostingsList returns the postings list for the specified term func (d *Dictionary) PostingsList(term []byte, except *roaring.Bitmap, prealloc segment.PostingsList) (segment.PostingsList, error) { @@ -96,140 +98,31 @@ func (d *Dictionary) postingsListInit(rv *PostingsList, except *roaring.Bitmap) } func (d *Dictionary) Contains(key []byte) (bool, error) { - return d.fst.Contains(key) -} - -// Iterator returns an iterator for this dictionary -func (d *Dictionary) Iterator() segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - if d.fst != nil { - itr, err := d.fst.Iterator(nil, nil) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } + return d.fst.Contains(key) } - - return rv -} - -// PrefixIterator returns an iterator which only visits terms having the -// the specified prefix -func (d *Dictionary) PrefixIterator(prefix string) segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - - kBeg := []byte(prefix) - kEnd := segment.IncrementBytes(kBeg) - - if d.fst != nil { - itr, err := d.fst.Iterator(kBeg, kEnd) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - } - - return rv -} - -// RangeIterator returns an iterator which only visits terms between the -// start and end terms. NOTE: bleve.index API specifies the end is inclusive. -func (d *Dictionary) RangeIterator(start, end string) segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - - // need to increment the end position to be inclusive - var endBytes []byte - if len(end) > 0 { - endBytes = []byte(end) - if endBytes[len(endBytes)-1] < 0xff { - endBytes[len(endBytes)-1]++ - } else { - endBytes = append(endBytes, 0xff) - } - } - - if d.fst != nil { - itr, err := d.fst.Iterator([]byte(start), endBytes) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - } - - return rv + return false, nil } // AutomatonIterator returns an iterator which only visits terms // having the the vellum automaton and start/end key range -func (d *Dictionary) AutomatonIterator(a vellum.Automaton, +func (d *Dictionary) AutomatonIterator(a segment.Automaton, startKeyInclusive, endKeyExclusive []byte) segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - if d.fst != nil { + rv := &DictionaryIterator{ + d: d, + } + itr, err := d.fst.Search(a, startKeyInclusive, endKeyExclusive) if err == nil { rv.itr = itr } else if err != vellum.ErrIteratorDone { rv.err = err } - } - return rv -} - -func (d *Dictionary) OnlyIterator(onlyTerms [][]byte, - includeCount bool) segment.DictionaryIterator { - - rv := &DictionaryIterator{ - d: d, - omitCount: !includeCount, - } - - var buf bytes.Buffer - builder, err := vellum.New(&buf, nil) - if err != nil { - rv.err = err return rv } - for _, term := range onlyTerms { - err = builder.Insert(term, 0) - if err != nil { - rv.err = err - return rv - } - } - err = builder.Close() - if err != nil { - rv.err = err - return rv - } - - onlyFST, err := vellum.Load(buf.Bytes()) - if err != nil { - rv.err = err - return rv - } - - itr, err := d.fst.Search(onlyFST, nil, nil) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - - return rv + return emptyDictionaryIterator } // DictionaryIterator is an iterator for term dictionary @@ -242,6 +135,8 @@ type DictionaryIterator struct { omitCount bool } +var emptyDictionaryIterator = &DictionaryIterator{} + // Next returns the next entry in the dictionary func (i *DictionaryIterator) Next() (*index.DictEntry, error) { if i.err != nil && i.err != vellum.ErrIteratorDone { diff --git a/vendor/github.com/blevesearch/zap/v11/docvalues.go b/vendor/github.com/blevesearch/zapx/v11/docvalues.go similarity index 93% rename from vendor/github.com/blevesearch/zap/v11/docvalues.go rename to vendor/github.com/blevesearch/zapx/v11/docvalues.go index 2566dc6d8c..27dba9962e 100644 --- a/vendor/github.com/blevesearch/zap/v11/docvalues.go +++ b/vendor/github.com/blevesearch/zapx/v11/docvalues.go @@ -22,9 +22,8 @@ import ( "reflect" "sort" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/blevesearch/bleve/size" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" "github.com/golang/snappy" ) @@ -53,9 +52,9 @@ type docValueReader struct { } func (di *docValueReader) size() int { - return reflectStaticSizedocValueReader + size.SizeOfPtr + + return reflectStaticSizedocValueReader + SizeOfPtr + len(di.field) + - len(di.chunkOffsets)*size.SizeOfUint64 + + len(di.chunkOffsets)*SizeOfUint64 + len(di.curChunkHeader)*reflectStaticSizeMetaData + len(di.curChunkData) } @@ -200,7 +199,7 @@ func (di *docValueReader) iterateAllDocValues(s *SegmentBase, visitor docNumTerm } func (di *docValueReader) visitDocValues(docNum uint64, - visitor index.DocumentFieldTermVisitor) error { + visitor index.DocValueVisitor) error { // binary search the term locations for the docNum start, end := di.getDocValueLocs(docNum) if start == math.MaxUint64 || end == math.MaxUint64 || start == end { @@ -246,10 +245,10 @@ func (di *docValueReader) getDocValueLocs(docNum uint64) (uint64, uint64) { return math.MaxUint64, math.MaxUint64 } -// VisitDocumentFieldTerms is an implementation of the -// DocumentFieldTermVisitable interface -func (s *SegmentBase) VisitDocumentFieldTerms(localDocNum uint64, fields []string, - visitor index.DocumentFieldTermVisitor, dvsIn segment.DocVisitState) ( +// VisitDocValues is an implementation of the +// DocValueVisitable interface +func (s *SegmentBase) VisitDocValues(localDocNum uint64, fields []string, + visitor index.DocValueVisitor, dvsIn segment.DocVisitState) ( segment.DocVisitState, error) { dvs, ok := dvsIn.(*docVisitState) if !ok || dvs == nil { diff --git a/vendor/github.com/blevesearch/zap/v11/enumerator.go b/vendor/github.com/blevesearch/zapx/v11/enumerator.go similarity index 99% rename from vendor/github.com/blevesearch/zap/v11/enumerator.go rename to vendor/github.com/blevesearch/zapx/v11/enumerator.go index cd6ff73c79..5531d2cf1d 100644 --- a/vendor/github.com/blevesearch/zap/v11/enumerator.go +++ b/vendor/github.com/blevesearch/zapx/v11/enumerator.go @@ -17,7 +17,7 @@ package zap import ( "bytes" - "github.com/couchbase/vellum" + "github.com/blevesearch/vellum" ) // enumerator provides an ordered traversal of multiple vellum diff --git a/vendor/github.com/blevesearch/zapx/v11/go.mod b/vendor/github.com/blevesearch/zapx/v11/go.mod new file mode 100644 index 0000000000..8866b0ff03 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v11/go.mod @@ -0,0 +1,13 @@ +module github.com/blevesearch/zapx/v11 + +go 1.12 + +require ( + github.com/RoaringBitmap/roaring v0.7.3 + github.com/blevesearch/bleve_index_api v1.0.1 + github.com/blevesearch/mmap-go v1.0.2 + github.com/blevesearch/scorch_segment_api/v2 v2.0.1 + github.com/blevesearch/vellum v1.0.5 + github.com/golang/snappy v0.0.1 + github.com/spf13/cobra v0.0.5 +) diff --git a/vendor/github.com/blevesearch/zapx/v11/go.sum b/vendor/github.com/blevesearch/zapx/v11/go.sum new file mode 100644 index 0000000000..b6351ec69e --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v11/go.sum @@ -0,0 +1,97 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= +github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= +github.com/RoaringBitmap/roaring v0.7.3 h1:RwirWpvFONt2EwHHEHhER7S4BHZkyj3qL5LXLlnQPZ4= +github.com/RoaringBitmap/roaring v0.7.3/go.mod h1:jdT9ykXwHFNdJbEtxePexlFYH9LXucApeS0/+/g+p1I= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= +github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/blevesearch/bleve_index_api v1.0.0/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= +github.com/blevesearch/bleve_index_api v1.0.1 h1:nx9++0hnyiGOHJwQQYfsUGzpRdEVE5LsylmmngQvaFk= +github.com/blevesearch/bleve_index_api v1.0.1/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= +github.com/blevesearch/mmap-go v1.0.2 h1:JtMHb+FgQCTTYIhtMvimw15dJwu1Y5lrZDMOFXVWPk0= +github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA= +github.com/blevesearch/scorch_segment_api/v2 v2.0.1 h1:fd+hPtZ8GsbqPK1HslGp7Vhoik4arZteA/IsCEgOisw= +github.com/blevesearch/scorch_segment_api/v2 v2.0.1/go.mod h1:lq7yK2jQy1yQjtjTfU931aVqz7pYxEudHaDwOt1tXfU= +github.com/blevesearch/vellum v1.0.5 h1:L5dJ7hKauRVbuH7I8uqLeSK92CPPY6FfrbAmLhAug8A= +github.com/blevesearch/vellum v1.0.5/go.mod h1:atE0EH3fvk43zzS7t1YNdNC7DbmcC3uz+eMD5xZ2OyQ= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dvyukov/go-fuzz v0.0.0-20210429054444-fca39067bc72/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= +github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= +github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= +github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= +github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200928182047-19e03678916f/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/vendor/github.com/blevesearch/zap/v11/intcoder.go b/vendor/github.com/blevesearch/zapx/v11/intcoder.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v11/intcoder.go rename to vendor/github.com/blevesearch/zapx/v11/intcoder.go diff --git a/vendor/github.com/blevesearch/zapx/v11/memuvarint.go b/vendor/github.com/blevesearch/zapx/v11/memuvarint.go new file mode 100644 index 0000000000..0c10c83a4e --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v11/memuvarint.go @@ -0,0 +1,94 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "fmt" +) + +type memUvarintReader struct { + C int // index of next byte to read from S + S []byte +} + +func newMemUvarintReader(s []byte) *memUvarintReader { + return &memUvarintReader{S: s} +} + +// Len returns the number of unread bytes. +func (r *memUvarintReader) Len() int { + n := len(r.S) - r.C + if n < 0 { + return 0 + } + return n +} + +// ReadUvarint reads an encoded uint64. The original code this was +// based on is at encoding/binary/ReadUvarint(). +func (r *memUvarintReader) ReadUvarint() (uint64, error) { + var x uint64 + var s uint + var C = r.C + var S = r.S + + for { + b := S[C] + C++ + + if b < 0x80 { + r.C = C + + // why 63? The original code had an 'i += 1' loop var and + // checked for i > 9 || i == 9 ...; but, we no longer + // check for the i var, but instead check here for s, + // which is incremented by 7. So, 7*9 == 63. + // + // why the "extra" >= check? The normal case is that s < + // 63, so we check this single >= guard first so that we + // hit the normal, nil-error return pathway sooner. + if s >= 63 && (s > 63 || s == 63 && b > 1) { + return 0, fmt.Errorf("memUvarintReader overflow") + } + + return x | uint64(b)< uint32 */ sizeOfUints := 16 - sizeInBytes := (len(s.path) + size.SizeOfString) + sizeOfUints + sizeInBytes := (len(s.path) + SizeOfString) + sizeOfUints // mutex, refs -> int64 sizeInBytes += 16 @@ -246,7 +245,7 @@ func (s *SegmentBase) loadFields() error { func (s *SegmentBase) Dictionary(field string) (segment.TermDictionary, error) { dict, err := s.dictionary(field) if err == nil && dict == nil { - return &segment.EmptyDictionary{}, nil + return emptyDictionary, nil } return dict, err } @@ -282,7 +281,6 @@ func (sb *SegmentBase) dictionary(field string) (rv *Dictionary, err error) { if err != nil { return nil, fmt.Errorf("dictionary field %s vellum reader err: %v", field, err) } - } } @@ -304,16 +302,16 @@ var visitDocumentCtxPool = sync.Pool{ }, } -// VisitDocument invokes the DocFieldValueVistor for each stored field +// VisitStoredFields invokes the StoredFieldValueVisitor for each stored field // for the specified doc number -func (s *SegmentBase) VisitDocument(num uint64, visitor segment.DocumentFieldValueVisitor) error { +func (s *SegmentBase) VisitStoredFields(num uint64, visitor segment.StoredFieldValueVisitor) error { vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) defer visitDocumentCtxPool.Put(vdc) - return s.visitDocument(vdc, num, visitor) + return s.visitStoredFields(vdc, num, visitor) } -func (s *SegmentBase) visitDocument(vdc *visitDocumentCtx, num uint64, - visitor segment.DocumentFieldValueVisitor) error { +func (s *SegmentBase) visitStoredFields(vdc *visitDocumentCtx, num uint64, + visitor segment.StoredFieldValueVisitor) error { // first make sure this is a valid number in this segment if num < s.numDocs { meta, compressed := s.getDocStoredMetaAndCompressed(num) diff --git a/vendor/github.com/blevesearch/zapx/v11/sizes.go b/vendor/github.com/blevesearch/zapx/v11/sizes.go new file mode 100644 index 0000000000..34166ea330 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v11/sizes.go @@ -0,0 +1,59 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "reflect" +) + +func init() { + var b bool + SizeOfBool = int(reflect.TypeOf(b).Size()) + var f32 float32 + SizeOfFloat32 = int(reflect.TypeOf(f32).Size()) + var f64 float64 + SizeOfFloat64 = int(reflect.TypeOf(f64).Size()) + var i int + SizeOfInt = int(reflect.TypeOf(i).Size()) + var m map[int]int + SizeOfMap = int(reflect.TypeOf(m).Size()) + var ptr *int + SizeOfPtr = int(reflect.TypeOf(ptr).Size()) + var slice []int + SizeOfSlice = int(reflect.TypeOf(slice).Size()) + var str string + SizeOfString = int(reflect.TypeOf(str).Size()) + var u8 uint8 + SizeOfUint8 = int(reflect.TypeOf(u8).Size()) + var u16 uint16 + SizeOfUint16 = int(reflect.TypeOf(u16).Size()) + var u32 uint32 + SizeOfUint32 = int(reflect.TypeOf(u32).Size()) + var u64 uint64 + SizeOfUint64 = int(reflect.TypeOf(u64).Size()) +} + +var SizeOfBool int +var SizeOfFloat32 int +var SizeOfFloat64 int +var SizeOfInt int +var SizeOfMap int +var SizeOfPtr int +var SizeOfSlice int +var SizeOfString int +var SizeOfUint8 int +var SizeOfUint16 int +var SizeOfUint32 int +var SizeOfUint64 int diff --git a/vendor/github.com/blevesearch/zap/v11/write.go b/vendor/github.com/blevesearch/zapx/v11/write.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v11/write.go rename to vendor/github.com/blevesearch/zapx/v11/write.go diff --git a/vendor/github.com/blevesearch/zap/v11/zap.md b/vendor/github.com/blevesearch/zapx/v11/zap.md similarity index 100% rename from vendor/github.com/blevesearch/zap/v11/zap.md rename to vendor/github.com/blevesearch/zapx/v11/zap.md diff --git a/vendor/github.com/blevesearch/zap/v12/.gitignore b/vendor/github.com/blevesearch/zapx/v12/.gitignore similarity index 100% rename from vendor/github.com/blevesearch/zap/v12/.gitignore rename to vendor/github.com/blevesearch/zapx/v12/.gitignore diff --git a/vendor/github.com/blevesearch/zapx/v12/.golangci.yml b/vendor/github.com/blevesearch/zapx/v12/.golangci.yml new file mode 100644 index 0000000000..f0f2f6067f --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v12/.golangci.yml @@ -0,0 +1,29 @@ +linters: + # please, do not use `enable-all`: it's deprecated and will be removed soon. + # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint + disable-all: true + enable: + - bodyclose + - deadcode + - depguard + - dupl + - errcheck + - gofmt + - goimports + - goprintffuncname + - gosec + - gosimple + - govet + - ineffassign + - interfacer + - misspell + - nakedret + - nolintlint + - rowserrcheck + - staticcheck + - structcheck + - typecheck + - unused + - varcheck + - whitespace + diff --git a/vendor/github.com/couchbase/vellum/LICENSE b/vendor/github.com/blevesearch/zapx/v12/LICENSE similarity index 100% rename from vendor/github.com/couchbase/vellum/LICENSE rename to vendor/github.com/blevesearch/zapx/v12/LICENSE diff --git a/vendor/github.com/blevesearch/zap/v11/README.md b/vendor/github.com/blevesearch/zapx/v12/README.md similarity index 94% rename from vendor/github.com/blevesearch/zap/v11/README.md rename to vendor/github.com/blevesearch/zapx/v12/README.md index 0facb669fd..4cbf1a145b 100644 --- a/vendor/github.com/blevesearch/zap/v11/README.md +++ b/vendor/github.com/blevesearch/zapx/v12/README.md @@ -1,4 +1,9 @@ -# zap file format +# zapx file format + +The zapx module is fork of [zap](https://github.com/blevesearch/zap) module which maintains file format compatibility, but removes dependency on bleve, and instead depends only on the indepenent interface modules: + +- [bleve_index_api](https://github.com/blevesearch/scorch_segment_api) +- [scorch_segment_api](https://github.com/blevesearch/scorch_segment_api) Advanced ZAP File Format Documentation is [here](zap.md). diff --git a/vendor/github.com/blevesearch/zap/v12/build.go b/vendor/github.com/blevesearch/zapx/v12/build.go similarity index 79% rename from vendor/github.com/blevesearch/zap/v12/build.go rename to vendor/github.com/blevesearch/zapx/v12/build.go index 467e5e007b..de8265c140 100644 --- a/vendor/github.com/blevesearch/zap/v12/build.go +++ b/vendor/github.com/blevesearch/zapx/v12/build.go @@ -16,10 +16,12 @@ package zap import ( "bufio" + "fmt" + "io" "math" "os" - "github.com/couchbase/vellum" + "github.com/blevesearch/vellum" ) const Version uint32 = 12 @@ -32,6 +34,16 @@ func (sb *SegmentBase) Persist(path string) error { return PersistSegmentBase(sb, path) } +// WriteTo is an implementation of io.WriterTo interface. +func (sb *SegmentBase) WriteTo(w io.Writer) (int64, error) { + if w == nil { + return 0, fmt.Errorf("invalid writer found") + } + + n, err := persistSegmentBaseToWriter(sb, w) + return int64(n), err +} + // PersistSegmentBase persists SegmentBase in the zap file format. func PersistSegmentBase(sb *SegmentBase, path string) error { flag := os.O_RDWR | os.O_CREATE @@ -46,22 +58,7 @@ func PersistSegmentBase(sb *SegmentBase, path string) error { _ = os.Remove(path) } - br := bufio.NewWriter(f) - - _, err = br.Write(sb.mem) - if err != nil { - cleanup() - return err - } - - err = persistFooter(sb.numDocs, sb.storedIndexOffset, sb.fieldsIndexOffset, sb.docValueOffset, - sb.chunkMode, sb.memCRC, br) - if err != nil { - cleanup() - return err - } - - err = br.Flush() + _, err = persistSegmentBaseToWriter(sb, f) if err != nil { cleanup() return err @@ -79,7 +76,40 @@ func PersistSegmentBase(sb *SegmentBase, path string) error { return err } - return nil + return err +} + +type bufWriter struct { + w *bufio.Writer + n int +} + +func (br *bufWriter) Write(in []byte) (int, error) { + n, err := br.w.Write(in) + br.n += n + return n, err +} + +func persistSegmentBaseToWriter(sb *SegmentBase, w io.Writer) (int, error) { + br := &bufWriter{w: bufio.NewWriter(w)} + + _, err := br.Write(sb.mem) + if err != nil { + return 0, err + } + + err = persistFooter(sb.numDocs, sb.storedIndexOffset, sb.fieldsIndexOffset, + sb.docValueOffset, sb.chunkMode, sb.memCRC, br) + if err != nil { + return 0, err + } + + err = br.w.Flush() + if err != nil { + return 0, err + } + + return br.n, nil } func persistStoredFieldValues(fieldID int, diff --git a/vendor/github.com/blevesearch/zap/v12/chunk.go b/vendor/github.com/blevesearch/zapx/v12/chunk.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v12/chunk.go rename to vendor/github.com/blevesearch/zapx/v12/chunk.go diff --git a/vendor/github.com/blevesearch/zap/v12/contentcoder.go b/vendor/github.com/blevesearch/zapx/v12/contentcoder.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v12/contentcoder.go rename to vendor/github.com/blevesearch/zapx/v12/contentcoder.go diff --git a/vendor/github.com/blevesearch/zap/v11/count.go b/vendor/github.com/blevesearch/zapx/v12/count.go similarity index 96% rename from vendor/github.com/blevesearch/zap/v11/count.go rename to vendor/github.com/blevesearch/zapx/v12/count.go index 50290f8882..b6135359fb 100644 --- a/vendor/github.com/blevesearch/zap/v11/count.go +++ b/vendor/github.com/blevesearch/zapx/v12/count.go @@ -18,7 +18,7 @@ import ( "hash/crc32" "io" - "github.com/blevesearch/bleve/index/scorch/segment" + segment "github.com/blevesearch/scorch_segment_api/v2" ) // CountHashWriter is a wrapper around a Writer which counts the number of diff --git a/vendor/github.com/blevesearch/zap/v11/dict.go b/vendor/github.com/blevesearch/zapx/v12/dict.go similarity index 60% rename from vendor/github.com/blevesearch/zap/v11/dict.go rename to vendor/github.com/blevesearch/zapx/v12/dict.go index ad4a8f8dc5..e30bf2420d 100644 --- a/vendor/github.com/blevesearch/zap/v11/dict.go +++ b/vendor/github.com/blevesearch/zapx/v12/dict.go @@ -15,13 +15,12 @@ package zap import ( - "bytes" "fmt" "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/couchbase/vellum" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" + "github.com/blevesearch/vellum" ) // Dictionary is the zap representation of the term dictionary @@ -33,6 +32,9 @@ type Dictionary struct { fstReader *vellum.Reader } +// represents an immutable, empty dictionary +var emptyDictionary = &Dictionary{} + // PostingsList returns the postings list for the specified term func (d *Dictionary) PostingsList(term []byte, except *roaring.Bitmap, prealloc segment.PostingsList) (segment.PostingsList, error) { @@ -96,140 +98,31 @@ func (d *Dictionary) postingsListInit(rv *PostingsList, except *roaring.Bitmap) } func (d *Dictionary) Contains(key []byte) (bool, error) { - return d.fst.Contains(key) -} - -// Iterator returns an iterator for this dictionary -func (d *Dictionary) Iterator() segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - if d.fst != nil { - itr, err := d.fst.Iterator(nil, nil) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } + return d.fst.Contains(key) } - - return rv -} - -// PrefixIterator returns an iterator which only visits terms having the -// the specified prefix -func (d *Dictionary) PrefixIterator(prefix string) segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - - kBeg := []byte(prefix) - kEnd := segment.IncrementBytes(kBeg) - - if d.fst != nil { - itr, err := d.fst.Iterator(kBeg, kEnd) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - } - - return rv -} - -// RangeIterator returns an iterator which only visits terms between the -// start and end terms. NOTE: bleve.index API specifies the end is inclusive. -func (d *Dictionary) RangeIterator(start, end string) segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - - // need to increment the end position to be inclusive - var endBytes []byte - if len(end) > 0 { - endBytes = []byte(end) - if endBytes[len(endBytes)-1] < 0xff { - endBytes[len(endBytes)-1]++ - } else { - endBytes = append(endBytes, 0xff) - } - } - - if d.fst != nil { - itr, err := d.fst.Iterator([]byte(start), endBytes) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - } - - return rv + return false, nil } // AutomatonIterator returns an iterator which only visits terms // having the the vellum automaton and start/end key range -func (d *Dictionary) AutomatonIterator(a vellum.Automaton, +func (d *Dictionary) AutomatonIterator(a segment.Automaton, startKeyInclusive, endKeyExclusive []byte) segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - if d.fst != nil { + rv := &DictionaryIterator{ + d: d, + } + itr, err := d.fst.Search(a, startKeyInclusive, endKeyExclusive) if err == nil { rv.itr = itr } else if err != vellum.ErrIteratorDone { rv.err = err } - } - return rv -} - -func (d *Dictionary) OnlyIterator(onlyTerms [][]byte, - includeCount bool) segment.DictionaryIterator { - - rv := &DictionaryIterator{ - d: d, - omitCount: !includeCount, - } - - var buf bytes.Buffer - builder, err := vellum.New(&buf, nil) - if err != nil { - rv.err = err return rv } - for _, term := range onlyTerms { - err = builder.Insert(term, 0) - if err != nil { - rv.err = err - return rv - } - } - err = builder.Close() - if err != nil { - rv.err = err - return rv - } - - onlyFST, err := vellum.Load(buf.Bytes()) - if err != nil { - rv.err = err - return rv - } - - itr, err := d.fst.Search(onlyFST, nil, nil) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - - return rv + return emptyDictionaryIterator } // DictionaryIterator is an iterator for term dictionary @@ -242,6 +135,8 @@ type DictionaryIterator struct { omitCount bool } +var emptyDictionaryIterator = &DictionaryIterator{} + // Next returns the next entry in the dictionary func (i *DictionaryIterator) Next() (*index.DictEntry, error) { if i.err != nil && i.err != vellum.ErrIteratorDone { diff --git a/vendor/github.com/blevesearch/zap/v15/docvalues.go b/vendor/github.com/blevesearch/zapx/v12/docvalues.go similarity index 94% rename from vendor/github.com/blevesearch/zap/v15/docvalues.go rename to vendor/github.com/blevesearch/zapx/v12/docvalues.go index 793797bd8a..a530aa5ad9 100644 --- a/vendor/github.com/blevesearch/zap/v15/docvalues.go +++ b/vendor/github.com/blevesearch/zapx/v12/docvalues.go @@ -22,9 +22,8 @@ import ( "reflect" "sort" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/blevesearch/bleve/size" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" "github.com/golang/snappy" ) @@ -53,9 +52,9 @@ type docValueReader struct { } func (di *docValueReader) size() int { - return reflectStaticSizedocValueReader + size.SizeOfPtr + + return reflectStaticSizedocValueReader + SizeOfPtr + len(di.field) + - len(di.chunkOffsets)*size.SizeOfUint64 + + len(di.chunkOffsets)*SizeOfUint64 + len(di.curChunkHeader)*reflectStaticSizeMetaData + len(di.curChunkData) } @@ -200,7 +199,7 @@ func (di *docValueReader) iterateAllDocValues(s *SegmentBase, visitor docNumTerm } func (di *docValueReader) visitDocValues(docNum uint64, - visitor index.DocumentFieldTermVisitor) error { + visitor index.DocValueVisitor) error { // binary search the term locations for the docNum start, end := di.getDocValueLocs(docNum) if start == math.MaxUint64 || end == math.MaxUint64 || start == end { @@ -246,10 +245,10 @@ func (di *docValueReader) getDocValueLocs(docNum uint64) (uint64, uint64) { return math.MaxUint64, math.MaxUint64 } -// VisitDocumentFieldTerms is an implementation of the -// DocumentFieldTermVisitable interface -func (s *SegmentBase) VisitDocumentFieldTerms(localDocNum uint64, fields []string, - visitor index.DocumentFieldTermVisitor, dvsIn segment.DocVisitState) ( +// VisitDocValues is an implementation of the +// DocValueVisitable interface +func (s *SegmentBase) VisitDocValues(localDocNum uint64, fields []string, + visitor index.DocValueVisitor, dvsIn segment.DocVisitState) ( segment.DocVisitState, error) { dvs, ok := dvsIn.(*docVisitState) if !ok || dvs == nil { diff --git a/vendor/github.com/blevesearch/zap/v12/enumerator.go b/vendor/github.com/blevesearch/zapx/v12/enumerator.go similarity index 99% rename from vendor/github.com/blevesearch/zap/v12/enumerator.go rename to vendor/github.com/blevesearch/zapx/v12/enumerator.go index bc5b7e6276..972a224165 100644 --- a/vendor/github.com/blevesearch/zap/v12/enumerator.go +++ b/vendor/github.com/blevesearch/zapx/v12/enumerator.go @@ -17,7 +17,7 @@ package zap import ( "bytes" - "github.com/couchbase/vellum" + "github.com/blevesearch/vellum" ) // enumerator provides an ordered traversal of multiple vellum diff --git a/vendor/github.com/blevesearch/zapx/v12/go.mod b/vendor/github.com/blevesearch/zapx/v12/go.mod new file mode 100644 index 0000000000..bbd3ab5e61 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v12/go.mod @@ -0,0 +1,13 @@ +module github.com/blevesearch/zapx/v12 + +go 1.12 + +require ( + github.com/RoaringBitmap/roaring v0.7.3 + github.com/blevesearch/bleve_index_api v1.0.1 + github.com/blevesearch/mmap-go v1.0.2 + github.com/blevesearch/scorch_segment_api/v2 v2.0.1 + github.com/blevesearch/vellum v1.0.5 + github.com/golang/snappy v0.0.1 + github.com/spf13/cobra v0.0.5 +) diff --git a/vendor/github.com/blevesearch/zapx/v12/go.sum b/vendor/github.com/blevesearch/zapx/v12/go.sum new file mode 100644 index 0000000000..b6351ec69e --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v12/go.sum @@ -0,0 +1,97 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= +github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= +github.com/RoaringBitmap/roaring v0.7.3 h1:RwirWpvFONt2EwHHEHhER7S4BHZkyj3qL5LXLlnQPZ4= +github.com/RoaringBitmap/roaring v0.7.3/go.mod h1:jdT9ykXwHFNdJbEtxePexlFYH9LXucApeS0/+/g+p1I= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= +github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/blevesearch/bleve_index_api v1.0.0/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= +github.com/blevesearch/bleve_index_api v1.0.1 h1:nx9++0hnyiGOHJwQQYfsUGzpRdEVE5LsylmmngQvaFk= +github.com/blevesearch/bleve_index_api v1.0.1/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= +github.com/blevesearch/mmap-go v1.0.2 h1:JtMHb+FgQCTTYIhtMvimw15dJwu1Y5lrZDMOFXVWPk0= +github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA= +github.com/blevesearch/scorch_segment_api/v2 v2.0.1 h1:fd+hPtZ8GsbqPK1HslGp7Vhoik4arZteA/IsCEgOisw= +github.com/blevesearch/scorch_segment_api/v2 v2.0.1/go.mod h1:lq7yK2jQy1yQjtjTfU931aVqz7pYxEudHaDwOt1tXfU= +github.com/blevesearch/vellum v1.0.5 h1:L5dJ7hKauRVbuH7I8uqLeSK92CPPY6FfrbAmLhAug8A= +github.com/blevesearch/vellum v1.0.5/go.mod h1:atE0EH3fvk43zzS7t1YNdNC7DbmcC3uz+eMD5xZ2OyQ= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dvyukov/go-fuzz v0.0.0-20210429054444-fca39067bc72/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= +github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= +github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= +github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= +github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200928182047-19e03678916f/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/vendor/github.com/blevesearch/zap/v12/intDecoder.go b/vendor/github.com/blevesearch/zapx/v12/intDecoder.go similarity index 92% rename from vendor/github.com/blevesearch/zap/v12/intDecoder.go rename to vendor/github.com/blevesearch/zapx/v12/intDecoder.go index 4cd008ff26..e968093149 100644 --- a/vendor/github.com/blevesearch/zap/v12/intDecoder.go +++ b/vendor/github.com/blevesearch/zapx/v12/intDecoder.go @@ -17,8 +17,6 @@ package zap import ( "encoding/binary" "fmt" - - "github.com/blevesearch/bleve/index/scorch/segment" ) type chunkedIntDecoder struct { @@ -27,7 +25,7 @@ type chunkedIntDecoder struct { chunkOffsets []uint64 curChunkBytes []byte data []byte - r *segment.MemUvarintReader + r *memUvarintReader } func newChunkedIntDecoder(buf []byte, offset uint64) *chunkedIntDecoder { @@ -56,7 +54,7 @@ func newChunkedIntDecoder(buf []byte, offset uint64) *chunkedIntDecoder { func (d *chunkedIntDecoder) loadChunk(chunk int) error { if d.startOffset == termNotEncoded { - d.r = segment.NewMemUvarintReader([]byte(nil)) + d.r = newMemUvarintReader([]byte(nil)) return nil } @@ -71,7 +69,7 @@ func (d *chunkedIntDecoder) loadChunk(chunk int) error { end += e d.curChunkBytes = d.data[start:end] if d.r == nil { - d.r = segment.NewMemUvarintReader(d.curChunkBytes) + d.r = newMemUvarintReader(d.curChunkBytes) } else { d.r.Reset(d.curChunkBytes) } diff --git a/vendor/github.com/blevesearch/zap/v12/intcoder.go b/vendor/github.com/blevesearch/zapx/v12/intcoder.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v12/intcoder.go rename to vendor/github.com/blevesearch/zapx/v12/intcoder.go diff --git a/vendor/github.com/blevesearch/zapx/v12/memuvarint.go b/vendor/github.com/blevesearch/zapx/v12/memuvarint.go new file mode 100644 index 0000000000..0c10c83a4e --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v12/memuvarint.go @@ -0,0 +1,94 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "fmt" +) + +type memUvarintReader struct { + C int // index of next byte to read from S + S []byte +} + +func newMemUvarintReader(s []byte) *memUvarintReader { + return &memUvarintReader{S: s} +} + +// Len returns the number of unread bytes. +func (r *memUvarintReader) Len() int { + n := len(r.S) - r.C + if n < 0 { + return 0 + } + return n +} + +// ReadUvarint reads an encoded uint64. The original code this was +// based on is at encoding/binary/ReadUvarint(). +func (r *memUvarintReader) ReadUvarint() (uint64, error) { + var x uint64 + var s uint + var C = r.C + var S = r.S + + for { + b := S[C] + C++ + + if b < 0x80 { + r.C = C + + // why 63? The original code had an 'i += 1' loop var and + // checked for i > 9 || i == 9 ...; but, we no longer + // check for the i var, but instead check here for s, + // which is incremented by 7. So, 7*9 == 63. + // + // why the "extra" >= check? The normal case is that s < + // 63, so we check this single >= guard first so that we + // hit the normal, nil-error return pathway sooner. + if s >= 63 && (s > 63 || s == 63 && b > 1) { + return 0, fmt.Errorf("memUvarintReader overflow") + } + + return x | uint64(b)< uint32 */ sizeOfUints := 16 - sizeInBytes := (len(s.path) + size.SizeOfString) + sizeOfUints + sizeInBytes := (len(s.path) + SizeOfString) + sizeOfUints // mutex, refs -> int64 sizeInBytes += 16 @@ -246,7 +245,7 @@ func (s *SegmentBase) loadFields() error { func (s *SegmentBase) Dictionary(field string) (segment.TermDictionary, error) { dict, err := s.dictionary(field) if err == nil && dict == nil { - return &segment.EmptyDictionary{}, nil + return emptyDictionary, nil } return dict, err } @@ -282,7 +281,6 @@ func (sb *SegmentBase) dictionary(field string) (rv *Dictionary, err error) { if err != nil { return nil, fmt.Errorf("dictionary field %s vellum reader err: %v", field, err) } - } } @@ -304,16 +302,16 @@ var visitDocumentCtxPool = sync.Pool{ }, } -// VisitDocument invokes the DocFieldValueVistor for each stored field +// VisitStoredFields invokes the StoredFieldValueVisitor for each stored field // for the specified doc number -func (s *SegmentBase) VisitDocument(num uint64, visitor segment.DocumentFieldValueVisitor) error { +func (s *SegmentBase) VisitStoredFields(num uint64, visitor segment.StoredFieldValueVisitor) error { vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) defer visitDocumentCtxPool.Put(vdc) - return s.visitDocument(vdc, num, visitor) + return s.visitStoredFields(vdc, num, visitor) } -func (s *SegmentBase) visitDocument(vdc *visitDocumentCtx, num uint64, - visitor segment.DocumentFieldValueVisitor) error { +func (s *SegmentBase) visitStoredFields(vdc *visitDocumentCtx, num uint64, + visitor segment.StoredFieldValueVisitor) error { // first make sure this is a valid number in this segment if num < s.numDocs { meta, compressed := s.getDocStoredMetaAndCompressed(num) diff --git a/vendor/github.com/blevesearch/zapx/v12/sizes.go b/vendor/github.com/blevesearch/zapx/v12/sizes.go new file mode 100644 index 0000000000..34166ea330 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v12/sizes.go @@ -0,0 +1,59 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "reflect" +) + +func init() { + var b bool + SizeOfBool = int(reflect.TypeOf(b).Size()) + var f32 float32 + SizeOfFloat32 = int(reflect.TypeOf(f32).Size()) + var f64 float64 + SizeOfFloat64 = int(reflect.TypeOf(f64).Size()) + var i int + SizeOfInt = int(reflect.TypeOf(i).Size()) + var m map[int]int + SizeOfMap = int(reflect.TypeOf(m).Size()) + var ptr *int + SizeOfPtr = int(reflect.TypeOf(ptr).Size()) + var slice []int + SizeOfSlice = int(reflect.TypeOf(slice).Size()) + var str string + SizeOfString = int(reflect.TypeOf(str).Size()) + var u8 uint8 + SizeOfUint8 = int(reflect.TypeOf(u8).Size()) + var u16 uint16 + SizeOfUint16 = int(reflect.TypeOf(u16).Size()) + var u32 uint32 + SizeOfUint32 = int(reflect.TypeOf(u32).Size()) + var u64 uint64 + SizeOfUint64 = int(reflect.TypeOf(u64).Size()) +} + +var SizeOfBool int +var SizeOfFloat32 int +var SizeOfFloat64 int +var SizeOfInt int +var SizeOfMap int +var SizeOfPtr int +var SizeOfSlice int +var SizeOfString int +var SizeOfUint8 int +var SizeOfUint16 int +var SizeOfUint32 int +var SizeOfUint64 int diff --git a/vendor/github.com/blevesearch/zap/v12/write.go b/vendor/github.com/blevesearch/zapx/v12/write.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v12/write.go rename to vendor/github.com/blevesearch/zapx/v12/write.go diff --git a/vendor/github.com/blevesearch/zap/v12/zap.md b/vendor/github.com/blevesearch/zapx/v12/zap.md similarity index 100% rename from vendor/github.com/blevesearch/zap/v12/zap.md rename to vendor/github.com/blevesearch/zapx/v12/zap.md diff --git a/vendor/github.com/blevesearch/zap/v13/.gitignore b/vendor/github.com/blevesearch/zapx/v13/.gitignore similarity index 100% rename from vendor/github.com/blevesearch/zap/v13/.gitignore rename to vendor/github.com/blevesearch/zapx/v13/.gitignore diff --git a/vendor/github.com/blevesearch/zapx/v13/.golangci.yml b/vendor/github.com/blevesearch/zapx/v13/.golangci.yml new file mode 100644 index 0000000000..f0f2f6067f --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v13/.golangci.yml @@ -0,0 +1,29 @@ +linters: + # please, do not use `enable-all`: it's deprecated and will be removed soon. + # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint + disable-all: true + enable: + - bodyclose + - deadcode + - depguard + - dupl + - errcheck + - gofmt + - goimports + - goprintffuncname + - gosec + - gosimple + - govet + - ineffassign + - interfacer + - misspell + - nakedret + - nolintlint + - rowserrcheck + - staticcheck + - structcheck + - typecheck + - unused + - varcheck + - whitespace + diff --git a/vendor/github.com/blevesearch/zapx/v13/LICENSE b/vendor/github.com/blevesearch/zapx/v13/LICENSE new file mode 100644 index 0000000000..7a4a3ea242 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v13/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/vendor/github.com/blevesearch/zap/v12/README.md b/vendor/github.com/blevesearch/zapx/v13/README.md similarity index 94% rename from vendor/github.com/blevesearch/zap/v12/README.md rename to vendor/github.com/blevesearch/zapx/v13/README.md index 0facb669fd..4cbf1a145b 100644 --- a/vendor/github.com/blevesearch/zap/v12/README.md +++ b/vendor/github.com/blevesearch/zapx/v13/README.md @@ -1,4 +1,9 @@ -# zap file format +# zapx file format + +The zapx module is fork of [zap](https://github.com/blevesearch/zap) module which maintains file format compatibility, but removes dependency on bleve, and instead depends only on the indepenent interface modules: + +- [bleve_index_api](https://github.com/blevesearch/scorch_segment_api) +- [scorch_segment_api](https://github.com/blevesearch/scorch_segment_api) Advanced ZAP File Format Documentation is [here](zap.md). diff --git a/vendor/github.com/blevesearch/zap/v13/build.go b/vendor/github.com/blevesearch/zapx/v13/build.go similarity index 79% rename from vendor/github.com/blevesearch/zap/v13/build.go rename to vendor/github.com/blevesearch/zapx/v13/build.go index 58d829f0c6..827e5c47e8 100644 --- a/vendor/github.com/blevesearch/zap/v13/build.go +++ b/vendor/github.com/blevesearch/zapx/v13/build.go @@ -16,10 +16,12 @@ package zap import ( "bufio" + "fmt" + "io" "math" "os" - "github.com/couchbase/vellum" + "github.com/blevesearch/vellum" ) const Version uint32 = 13 @@ -32,6 +34,16 @@ func (sb *SegmentBase) Persist(path string) error { return PersistSegmentBase(sb, path) } +// WriteTo is an implementation of io.WriterTo interface. +func (sb *SegmentBase) WriteTo(w io.Writer) (int64, error) { + if w == nil { + return 0, fmt.Errorf("invalid writer found") + } + + n, err := persistSegmentBaseToWriter(sb, w) + return int64(n), err +} + // PersistSegmentBase persists SegmentBase in the zap file format. func PersistSegmentBase(sb *SegmentBase, path string) error { flag := os.O_RDWR | os.O_CREATE @@ -46,22 +58,7 @@ func PersistSegmentBase(sb *SegmentBase, path string) error { _ = os.Remove(path) } - br := bufio.NewWriter(f) - - _, err = br.Write(sb.mem) - if err != nil { - cleanup() - return err - } - - err = persistFooter(sb.numDocs, sb.storedIndexOffset, sb.fieldsIndexOffset, sb.docValueOffset, - sb.chunkMode, sb.memCRC, br) - if err != nil { - cleanup() - return err - } - - err = br.Flush() + _, err = persistSegmentBaseToWriter(sb, f) if err != nil { cleanup() return err @@ -79,7 +76,40 @@ func PersistSegmentBase(sb *SegmentBase, path string) error { return err } - return nil + return err +} + +type bufWriter struct { + w *bufio.Writer + n int +} + +func (br *bufWriter) Write(in []byte) (int, error) { + n, err := br.w.Write(in) + br.n += n + return n, err +} + +func persistSegmentBaseToWriter(sb *SegmentBase, w io.Writer) (int, error) { + br := &bufWriter{w: bufio.NewWriter(w)} + + _, err := br.Write(sb.mem) + if err != nil { + return 0, err + } + + err = persistFooter(sb.numDocs, sb.storedIndexOffset, sb.fieldsIndexOffset, + sb.docValueOffset, sb.chunkMode, sb.memCRC, br) + if err != nil { + return 0, err + } + + err = br.w.Flush() + if err != nil { + return 0, err + } + + return br.n, nil } func persistStoredFieldValues(fieldID int, diff --git a/vendor/github.com/blevesearch/zap/v13/chunk.go b/vendor/github.com/blevesearch/zapx/v13/chunk.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v13/chunk.go rename to vendor/github.com/blevesearch/zapx/v13/chunk.go diff --git a/vendor/github.com/blevesearch/zap/v13/contentcoder.go b/vendor/github.com/blevesearch/zapx/v13/contentcoder.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v13/contentcoder.go rename to vendor/github.com/blevesearch/zapx/v13/contentcoder.go diff --git a/vendor/github.com/blevesearch/zap/v13/count.go b/vendor/github.com/blevesearch/zapx/v13/count.go similarity index 96% rename from vendor/github.com/blevesearch/zap/v13/count.go rename to vendor/github.com/blevesearch/zapx/v13/count.go index 50290f8882..b6135359fb 100644 --- a/vendor/github.com/blevesearch/zap/v13/count.go +++ b/vendor/github.com/blevesearch/zapx/v13/count.go @@ -18,7 +18,7 @@ import ( "hash/crc32" "io" - "github.com/blevesearch/bleve/index/scorch/segment" + segment "github.com/blevesearch/scorch_segment_api/v2" ) // CountHashWriter is a wrapper around a Writer which counts the number of diff --git a/vendor/github.com/blevesearch/zap/v12/dict.go b/vendor/github.com/blevesearch/zapx/v13/dict.go similarity index 60% rename from vendor/github.com/blevesearch/zap/v12/dict.go rename to vendor/github.com/blevesearch/zapx/v13/dict.go index ad4a8f8dc5..e30bf2420d 100644 --- a/vendor/github.com/blevesearch/zap/v12/dict.go +++ b/vendor/github.com/blevesearch/zapx/v13/dict.go @@ -15,13 +15,12 @@ package zap import ( - "bytes" "fmt" "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/couchbase/vellum" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" + "github.com/blevesearch/vellum" ) // Dictionary is the zap representation of the term dictionary @@ -33,6 +32,9 @@ type Dictionary struct { fstReader *vellum.Reader } +// represents an immutable, empty dictionary +var emptyDictionary = &Dictionary{} + // PostingsList returns the postings list for the specified term func (d *Dictionary) PostingsList(term []byte, except *roaring.Bitmap, prealloc segment.PostingsList) (segment.PostingsList, error) { @@ -96,140 +98,31 @@ func (d *Dictionary) postingsListInit(rv *PostingsList, except *roaring.Bitmap) } func (d *Dictionary) Contains(key []byte) (bool, error) { - return d.fst.Contains(key) -} - -// Iterator returns an iterator for this dictionary -func (d *Dictionary) Iterator() segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - if d.fst != nil { - itr, err := d.fst.Iterator(nil, nil) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } + return d.fst.Contains(key) } - - return rv -} - -// PrefixIterator returns an iterator which only visits terms having the -// the specified prefix -func (d *Dictionary) PrefixIterator(prefix string) segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - - kBeg := []byte(prefix) - kEnd := segment.IncrementBytes(kBeg) - - if d.fst != nil { - itr, err := d.fst.Iterator(kBeg, kEnd) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - } - - return rv -} - -// RangeIterator returns an iterator which only visits terms between the -// start and end terms. NOTE: bleve.index API specifies the end is inclusive. -func (d *Dictionary) RangeIterator(start, end string) segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - - // need to increment the end position to be inclusive - var endBytes []byte - if len(end) > 0 { - endBytes = []byte(end) - if endBytes[len(endBytes)-1] < 0xff { - endBytes[len(endBytes)-1]++ - } else { - endBytes = append(endBytes, 0xff) - } - } - - if d.fst != nil { - itr, err := d.fst.Iterator([]byte(start), endBytes) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - } - - return rv + return false, nil } // AutomatonIterator returns an iterator which only visits terms // having the the vellum automaton and start/end key range -func (d *Dictionary) AutomatonIterator(a vellum.Automaton, +func (d *Dictionary) AutomatonIterator(a segment.Automaton, startKeyInclusive, endKeyExclusive []byte) segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - if d.fst != nil { + rv := &DictionaryIterator{ + d: d, + } + itr, err := d.fst.Search(a, startKeyInclusive, endKeyExclusive) if err == nil { rv.itr = itr } else if err != vellum.ErrIteratorDone { rv.err = err } - } - return rv -} - -func (d *Dictionary) OnlyIterator(onlyTerms [][]byte, - includeCount bool) segment.DictionaryIterator { - - rv := &DictionaryIterator{ - d: d, - omitCount: !includeCount, - } - - var buf bytes.Buffer - builder, err := vellum.New(&buf, nil) - if err != nil { - rv.err = err return rv } - for _, term := range onlyTerms { - err = builder.Insert(term, 0) - if err != nil { - rv.err = err - return rv - } - } - err = builder.Close() - if err != nil { - rv.err = err - return rv - } - - onlyFST, err := vellum.Load(buf.Bytes()) - if err != nil { - rv.err = err - return rv - } - - itr, err := d.fst.Search(onlyFST, nil, nil) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - - return rv + return emptyDictionaryIterator } // DictionaryIterator is an iterator for term dictionary @@ -242,6 +135,8 @@ type DictionaryIterator struct { omitCount bool } +var emptyDictionaryIterator = &DictionaryIterator{} + // Next returns the next entry in the dictionary func (i *DictionaryIterator) Next() (*index.DictEntry, error) { if i.err != nil && i.err != vellum.ErrIteratorDone { diff --git a/vendor/github.com/blevesearch/zap/v12/docvalues.go b/vendor/github.com/blevesearch/zapx/v13/docvalues.go similarity index 94% rename from vendor/github.com/blevesearch/zap/v12/docvalues.go rename to vendor/github.com/blevesearch/zapx/v13/docvalues.go index 793797bd8a..a530aa5ad9 100644 --- a/vendor/github.com/blevesearch/zap/v12/docvalues.go +++ b/vendor/github.com/blevesearch/zapx/v13/docvalues.go @@ -22,9 +22,8 @@ import ( "reflect" "sort" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/blevesearch/bleve/size" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" "github.com/golang/snappy" ) @@ -53,9 +52,9 @@ type docValueReader struct { } func (di *docValueReader) size() int { - return reflectStaticSizedocValueReader + size.SizeOfPtr + + return reflectStaticSizedocValueReader + SizeOfPtr + len(di.field) + - len(di.chunkOffsets)*size.SizeOfUint64 + + len(di.chunkOffsets)*SizeOfUint64 + len(di.curChunkHeader)*reflectStaticSizeMetaData + len(di.curChunkData) } @@ -200,7 +199,7 @@ func (di *docValueReader) iterateAllDocValues(s *SegmentBase, visitor docNumTerm } func (di *docValueReader) visitDocValues(docNum uint64, - visitor index.DocumentFieldTermVisitor) error { + visitor index.DocValueVisitor) error { // binary search the term locations for the docNum start, end := di.getDocValueLocs(docNum) if start == math.MaxUint64 || end == math.MaxUint64 || start == end { @@ -246,10 +245,10 @@ func (di *docValueReader) getDocValueLocs(docNum uint64) (uint64, uint64) { return math.MaxUint64, math.MaxUint64 } -// VisitDocumentFieldTerms is an implementation of the -// DocumentFieldTermVisitable interface -func (s *SegmentBase) VisitDocumentFieldTerms(localDocNum uint64, fields []string, - visitor index.DocumentFieldTermVisitor, dvsIn segment.DocVisitState) ( +// VisitDocValues is an implementation of the +// DocValueVisitable interface +func (s *SegmentBase) VisitDocValues(localDocNum uint64, fields []string, + visitor index.DocValueVisitor, dvsIn segment.DocVisitState) ( segment.DocVisitState, error) { dvs, ok := dvsIn.(*docVisitState) if !ok || dvs == nil { diff --git a/vendor/github.com/blevesearch/zap/v14/enumerator.go b/vendor/github.com/blevesearch/zapx/v13/enumerator.go similarity index 99% rename from vendor/github.com/blevesearch/zap/v14/enumerator.go rename to vendor/github.com/blevesearch/zapx/v13/enumerator.go index bc5b7e6276..972a224165 100644 --- a/vendor/github.com/blevesearch/zap/v14/enumerator.go +++ b/vendor/github.com/blevesearch/zapx/v13/enumerator.go @@ -17,7 +17,7 @@ package zap import ( "bytes" - "github.com/couchbase/vellum" + "github.com/blevesearch/vellum" ) // enumerator provides an ordered traversal of multiple vellum diff --git a/vendor/github.com/blevesearch/zapx/v13/go.mod b/vendor/github.com/blevesearch/zapx/v13/go.mod new file mode 100644 index 0000000000..5214076fba --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v13/go.mod @@ -0,0 +1,13 @@ +module github.com/blevesearch/zapx/v13 + +go 1.12 + +require ( + github.com/RoaringBitmap/roaring v0.7.3 + github.com/blevesearch/bleve_index_api v1.0.1 + github.com/blevesearch/mmap-go v1.0.2 + github.com/blevesearch/scorch_segment_api/v2 v2.0.1 + github.com/blevesearch/vellum v1.0.5 + github.com/golang/snappy v0.0.1 + github.com/spf13/cobra v0.0.5 +) diff --git a/vendor/github.com/blevesearch/zapx/v13/go.sum b/vendor/github.com/blevesearch/zapx/v13/go.sum new file mode 100644 index 0000000000..b6351ec69e --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v13/go.sum @@ -0,0 +1,97 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= +github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= +github.com/RoaringBitmap/roaring v0.7.3 h1:RwirWpvFONt2EwHHEHhER7S4BHZkyj3qL5LXLlnQPZ4= +github.com/RoaringBitmap/roaring v0.7.3/go.mod h1:jdT9ykXwHFNdJbEtxePexlFYH9LXucApeS0/+/g+p1I= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= +github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/blevesearch/bleve_index_api v1.0.0/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= +github.com/blevesearch/bleve_index_api v1.0.1 h1:nx9++0hnyiGOHJwQQYfsUGzpRdEVE5LsylmmngQvaFk= +github.com/blevesearch/bleve_index_api v1.0.1/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= +github.com/blevesearch/mmap-go v1.0.2 h1:JtMHb+FgQCTTYIhtMvimw15dJwu1Y5lrZDMOFXVWPk0= +github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA= +github.com/blevesearch/scorch_segment_api/v2 v2.0.1 h1:fd+hPtZ8GsbqPK1HslGp7Vhoik4arZteA/IsCEgOisw= +github.com/blevesearch/scorch_segment_api/v2 v2.0.1/go.mod h1:lq7yK2jQy1yQjtjTfU931aVqz7pYxEudHaDwOt1tXfU= +github.com/blevesearch/vellum v1.0.5 h1:L5dJ7hKauRVbuH7I8uqLeSK92CPPY6FfrbAmLhAug8A= +github.com/blevesearch/vellum v1.0.5/go.mod h1:atE0EH3fvk43zzS7t1YNdNC7DbmcC3uz+eMD5xZ2OyQ= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dvyukov/go-fuzz v0.0.0-20210429054444-fca39067bc72/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= +github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= +github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= +github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= +github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200928182047-19e03678916f/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/vendor/github.com/blevesearch/zap/v13/intDecoder.go b/vendor/github.com/blevesearch/zapx/v13/intDecoder.go similarity index 92% rename from vendor/github.com/blevesearch/zap/v13/intDecoder.go rename to vendor/github.com/blevesearch/zapx/v13/intDecoder.go index 4cd008ff26..e968093149 100644 --- a/vendor/github.com/blevesearch/zap/v13/intDecoder.go +++ b/vendor/github.com/blevesearch/zapx/v13/intDecoder.go @@ -17,8 +17,6 @@ package zap import ( "encoding/binary" "fmt" - - "github.com/blevesearch/bleve/index/scorch/segment" ) type chunkedIntDecoder struct { @@ -27,7 +25,7 @@ type chunkedIntDecoder struct { chunkOffsets []uint64 curChunkBytes []byte data []byte - r *segment.MemUvarintReader + r *memUvarintReader } func newChunkedIntDecoder(buf []byte, offset uint64) *chunkedIntDecoder { @@ -56,7 +54,7 @@ func newChunkedIntDecoder(buf []byte, offset uint64) *chunkedIntDecoder { func (d *chunkedIntDecoder) loadChunk(chunk int) error { if d.startOffset == termNotEncoded { - d.r = segment.NewMemUvarintReader([]byte(nil)) + d.r = newMemUvarintReader([]byte(nil)) return nil } @@ -71,7 +69,7 @@ func (d *chunkedIntDecoder) loadChunk(chunk int) error { end += e d.curChunkBytes = d.data[start:end] if d.r == nil { - d.r = segment.NewMemUvarintReader(d.curChunkBytes) + d.r = newMemUvarintReader(d.curChunkBytes) } else { d.r.Reset(d.curChunkBytes) } diff --git a/vendor/github.com/blevesearch/zap/v13/intcoder.go b/vendor/github.com/blevesearch/zapx/v13/intcoder.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v13/intcoder.go rename to vendor/github.com/blevesearch/zapx/v13/intcoder.go diff --git a/vendor/github.com/blevesearch/zapx/v13/memuvarint.go b/vendor/github.com/blevesearch/zapx/v13/memuvarint.go new file mode 100644 index 0000000000..0c10c83a4e --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v13/memuvarint.go @@ -0,0 +1,94 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "fmt" +) + +type memUvarintReader struct { + C int // index of next byte to read from S + S []byte +} + +func newMemUvarintReader(s []byte) *memUvarintReader { + return &memUvarintReader{S: s} +} + +// Len returns the number of unread bytes. +func (r *memUvarintReader) Len() int { + n := len(r.S) - r.C + if n < 0 { + return 0 + } + return n +} + +// ReadUvarint reads an encoded uint64. The original code this was +// based on is at encoding/binary/ReadUvarint(). +func (r *memUvarintReader) ReadUvarint() (uint64, error) { + var x uint64 + var s uint + var C = r.C + var S = r.S + + for { + b := S[C] + C++ + + if b < 0x80 { + r.C = C + + // why 63? The original code had an 'i += 1' loop var and + // checked for i > 9 || i == 9 ...; but, we no longer + // check for the i var, but instead check here for s, + // which is incremented by 7. So, 7*9 == 63. + // + // why the "extra" >= check? The normal case is that s < + // 63, so we check this single >= guard first so that we + // hit the normal, nil-error return pathway sooner. + if s >= 63 && (s > 63 || s == 63 && b > 1) { + return 0, fmt.Errorf("memUvarintReader overflow") + } + + return x | uint64(b)< uint32 */ sizeOfUints := 16 - sizeInBytes := (len(s.path) + size.SizeOfString) + sizeOfUints + sizeInBytes := (len(s.path) + SizeOfString) + sizeOfUints // mutex, refs -> int64 sizeInBytes += 16 @@ -246,7 +245,7 @@ func (s *SegmentBase) loadFields() error { func (s *SegmentBase) Dictionary(field string) (segment.TermDictionary, error) { dict, err := s.dictionary(field) if err == nil && dict == nil { - return &segment.EmptyDictionary{}, nil + return emptyDictionary, nil } return dict, err } @@ -282,7 +281,6 @@ func (sb *SegmentBase) dictionary(field string) (rv *Dictionary, err error) { if err != nil { return nil, fmt.Errorf("dictionary field %s vellum reader err: %v", field, err) } - } } @@ -304,16 +302,16 @@ var visitDocumentCtxPool = sync.Pool{ }, } -// VisitDocument invokes the DocFieldValueVistor for each stored field +// VisitStoredFields invokes the StoredFieldValueVisitor for each stored field // for the specified doc number -func (s *SegmentBase) VisitDocument(num uint64, visitor segment.DocumentFieldValueVisitor) error { +func (s *SegmentBase) VisitStoredFields(num uint64, visitor segment.StoredFieldValueVisitor) error { vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) defer visitDocumentCtxPool.Put(vdc) - return s.visitDocument(vdc, num, visitor) + return s.visitStoredFields(vdc, num, visitor) } -func (s *SegmentBase) visitDocument(vdc *visitDocumentCtx, num uint64, - visitor segment.DocumentFieldValueVisitor) error { +func (s *SegmentBase) visitStoredFields(vdc *visitDocumentCtx, num uint64, + visitor segment.StoredFieldValueVisitor) error { // first make sure this is a valid number in this segment if num < s.numDocs { meta, compressed := s.getDocStoredMetaAndCompressed(num) diff --git a/vendor/github.com/blevesearch/zapx/v13/sizes.go b/vendor/github.com/blevesearch/zapx/v13/sizes.go new file mode 100644 index 0000000000..34166ea330 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v13/sizes.go @@ -0,0 +1,59 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "reflect" +) + +func init() { + var b bool + SizeOfBool = int(reflect.TypeOf(b).Size()) + var f32 float32 + SizeOfFloat32 = int(reflect.TypeOf(f32).Size()) + var f64 float64 + SizeOfFloat64 = int(reflect.TypeOf(f64).Size()) + var i int + SizeOfInt = int(reflect.TypeOf(i).Size()) + var m map[int]int + SizeOfMap = int(reflect.TypeOf(m).Size()) + var ptr *int + SizeOfPtr = int(reflect.TypeOf(ptr).Size()) + var slice []int + SizeOfSlice = int(reflect.TypeOf(slice).Size()) + var str string + SizeOfString = int(reflect.TypeOf(str).Size()) + var u8 uint8 + SizeOfUint8 = int(reflect.TypeOf(u8).Size()) + var u16 uint16 + SizeOfUint16 = int(reflect.TypeOf(u16).Size()) + var u32 uint32 + SizeOfUint32 = int(reflect.TypeOf(u32).Size()) + var u64 uint64 + SizeOfUint64 = int(reflect.TypeOf(u64).Size()) +} + +var SizeOfBool int +var SizeOfFloat32 int +var SizeOfFloat64 int +var SizeOfInt int +var SizeOfMap int +var SizeOfPtr int +var SizeOfSlice int +var SizeOfString int +var SizeOfUint8 int +var SizeOfUint16 int +var SizeOfUint32 int +var SizeOfUint64 int diff --git a/vendor/github.com/blevesearch/zap/v13/write.go b/vendor/github.com/blevesearch/zapx/v13/write.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v13/write.go rename to vendor/github.com/blevesearch/zapx/v13/write.go diff --git a/vendor/github.com/blevesearch/zap/v13/zap.md b/vendor/github.com/blevesearch/zapx/v13/zap.md similarity index 100% rename from vendor/github.com/blevesearch/zap/v13/zap.md rename to vendor/github.com/blevesearch/zapx/v13/zap.md diff --git a/vendor/github.com/blevesearch/zap/v14/.gitignore b/vendor/github.com/blevesearch/zapx/v14/.gitignore similarity index 100% rename from vendor/github.com/blevesearch/zap/v14/.gitignore rename to vendor/github.com/blevesearch/zapx/v14/.gitignore diff --git a/vendor/github.com/blevesearch/zapx/v14/.golangci.yml b/vendor/github.com/blevesearch/zapx/v14/.golangci.yml new file mode 100644 index 0000000000..f0f2f6067f --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v14/.golangci.yml @@ -0,0 +1,29 @@ +linters: + # please, do not use `enable-all`: it's deprecated and will be removed soon. + # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint + disable-all: true + enable: + - bodyclose + - deadcode + - depguard + - dupl + - errcheck + - gofmt + - goimports + - goprintffuncname + - gosec + - gosimple + - govet + - ineffassign + - interfacer + - misspell + - nakedret + - nolintlint + - rowserrcheck + - staticcheck + - structcheck + - typecheck + - unused + - varcheck + - whitespace + diff --git a/vendor/github.com/blevesearch/zapx/v14/LICENSE b/vendor/github.com/blevesearch/zapx/v14/LICENSE new file mode 100644 index 0000000000..7a4a3ea242 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v14/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/vendor/github.com/blevesearch/zap/v13/README.md b/vendor/github.com/blevesearch/zapx/v14/README.md similarity index 94% rename from vendor/github.com/blevesearch/zap/v13/README.md rename to vendor/github.com/blevesearch/zapx/v14/README.md index 0facb669fd..4cbf1a145b 100644 --- a/vendor/github.com/blevesearch/zap/v13/README.md +++ b/vendor/github.com/blevesearch/zapx/v14/README.md @@ -1,4 +1,9 @@ -# zap file format +# zapx file format + +The zapx module is fork of [zap](https://github.com/blevesearch/zap) module which maintains file format compatibility, but removes dependency on bleve, and instead depends only on the indepenent interface modules: + +- [bleve_index_api](https://github.com/blevesearch/scorch_segment_api) +- [scorch_segment_api](https://github.com/blevesearch/scorch_segment_api) Advanced ZAP File Format Documentation is [here](zap.md). diff --git a/vendor/github.com/blevesearch/zap/v14/build.go b/vendor/github.com/blevesearch/zapx/v14/build.go similarity index 79% rename from vendor/github.com/blevesearch/zap/v14/build.go rename to vendor/github.com/blevesearch/zapx/v14/build.go index 7a8dce0dcc..b36878abbb 100644 --- a/vendor/github.com/blevesearch/zap/v14/build.go +++ b/vendor/github.com/blevesearch/zapx/v14/build.go @@ -16,10 +16,12 @@ package zap import ( "bufio" + "fmt" + "io" "math" "os" - "github.com/couchbase/vellum" + "github.com/blevesearch/vellum" ) const Version uint32 = 14 @@ -32,6 +34,16 @@ func (sb *SegmentBase) Persist(path string) error { return PersistSegmentBase(sb, path) } +// WriteTo is an implementation of io.WriterTo interface. +func (sb *SegmentBase) WriteTo(w io.Writer) (int64, error) { + if w == nil { + return 0, fmt.Errorf("invalid writer found") + } + + n, err := persistSegmentBaseToWriter(sb, w) + return int64(n), err +} + // PersistSegmentBase persists SegmentBase in the zap file format. func PersistSegmentBase(sb *SegmentBase, path string) error { flag := os.O_RDWR | os.O_CREATE @@ -46,22 +58,7 @@ func PersistSegmentBase(sb *SegmentBase, path string) error { _ = os.Remove(path) } - br := bufio.NewWriter(f) - - _, err = br.Write(sb.mem) - if err != nil { - cleanup() - return err - } - - err = persistFooter(sb.numDocs, sb.storedIndexOffset, sb.fieldsIndexOffset, sb.docValueOffset, - sb.chunkMode, sb.memCRC, br) - if err != nil { - cleanup() - return err - } - - err = br.Flush() + _, err = persistSegmentBaseToWriter(sb, f) if err != nil { cleanup() return err @@ -79,7 +76,40 @@ func PersistSegmentBase(sb *SegmentBase, path string) error { return err } - return nil + return err +} + +type bufWriter struct { + w *bufio.Writer + n int +} + +func (br *bufWriter) Write(in []byte) (int, error) { + n, err := br.w.Write(in) + br.n += n + return n, err +} + +func persistSegmentBaseToWriter(sb *SegmentBase, w io.Writer) (int, error) { + br := &bufWriter{w: bufio.NewWriter(w)} + + _, err := br.Write(sb.mem) + if err != nil { + return 0, err + } + + err = persistFooter(sb.numDocs, sb.storedIndexOffset, sb.fieldsIndexOffset, + sb.docValueOffset, sb.chunkMode, sb.memCRC, br) + if err != nil { + return 0, err + } + + err = br.w.Flush() + if err != nil { + return 0, err + } + + return br.n, nil } func persistStoredFieldValues(fieldID int, diff --git a/vendor/github.com/blevesearch/zap/v14/chunk.go b/vendor/github.com/blevesearch/zapx/v14/chunk.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v14/chunk.go rename to vendor/github.com/blevesearch/zapx/v14/chunk.go diff --git a/vendor/github.com/blevesearch/zap/v14/contentcoder.go b/vendor/github.com/blevesearch/zapx/v14/contentcoder.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v14/contentcoder.go rename to vendor/github.com/blevesearch/zapx/v14/contentcoder.go diff --git a/vendor/github.com/blevesearch/zap/v14/count.go b/vendor/github.com/blevesearch/zapx/v14/count.go similarity index 96% rename from vendor/github.com/blevesearch/zap/v14/count.go rename to vendor/github.com/blevesearch/zapx/v14/count.go index 50290f8882..b6135359fb 100644 --- a/vendor/github.com/blevesearch/zap/v14/count.go +++ b/vendor/github.com/blevesearch/zapx/v14/count.go @@ -18,7 +18,7 @@ import ( "hash/crc32" "io" - "github.com/blevesearch/bleve/index/scorch/segment" + segment "github.com/blevesearch/scorch_segment_api/v2" ) // CountHashWriter is a wrapper around a Writer which counts the number of diff --git a/vendor/github.com/blevesearch/zap/v14/dict.go b/vendor/github.com/blevesearch/zapx/v14/dict.go similarity index 60% rename from vendor/github.com/blevesearch/zap/v14/dict.go rename to vendor/github.com/blevesearch/zapx/v14/dict.go index ad4a8f8dc5..e30bf2420d 100644 --- a/vendor/github.com/blevesearch/zap/v14/dict.go +++ b/vendor/github.com/blevesearch/zapx/v14/dict.go @@ -15,13 +15,12 @@ package zap import ( - "bytes" "fmt" "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/couchbase/vellum" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" + "github.com/blevesearch/vellum" ) // Dictionary is the zap representation of the term dictionary @@ -33,6 +32,9 @@ type Dictionary struct { fstReader *vellum.Reader } +// represents an immutable, empty dictionary +var emptyDictionary = &Dictionary{} + // PostingsList returns the postings list for the specified term func (d *Dictionary) PostingsList(term []byte, except *roaring.Bitmap, prealloc segment.PostingsList) (segment.PostingsList, error) { @@ -96,140 +98,31 @@ func (d *Dictionary) postingsListInit(rv *PostingsList, except *roaring.Bitmap) } func (d *Dictionary) Contains(key []byte) (bool, error) { - return d.fst.Contains(key) -} - -// Iterator returns an iterator for this dictionary -func (d *Dictionary) Iterator() segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - if d.fst != nil { - itr, err := d.fst.Iterator(nil, nil) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } + return d.fst.Contains(key) } - - return rv -} - -// PrefixIterator returns an iterator which only visits terms having the -// the specified prefix -func (d *Dictionary) PrefixIterator(prefix string) segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - - kBeg := []byte(prefix) - kEnd := segment.IncrementBytes(kBeg) - - if d.fst != nil { - itr, err := d.fst.Iterator(kBeg, kEnd) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - } - - return rv -} - -// RangeIterator returns an iterator which only visits terms between the -// start and end terms. NOTE: bleve.index API specifies the end is inclusive. -func (d *Dictionary) RangeIterator(start, end string) segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - - // need to increment the end position to be inclusive - var endBytes []byte - if len(end) > 0 { - endBytes = []byte(end) - if endBytes[len(endBytes)-1] < 0xff { - endBytes[len(endBytes)-1]++ - } else { - endBytes = append(endBytes, 0xff) - } - } - - if d.fst != nil { - itr, err := d.fst.Iterator([]byte(start), endBytes) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - } - - return rv + return false, nil } // AutomatonIterator returns an iterator which only visits terms // having the the vellum automaton and start/end key range -func (d *Dictionary) AutomatonIterator(a vellum.Automaton, +func (d *Dictionary) AutomatonIterator(a segment.Automaton, startKeyInclusive, endKeyExclusive []byte) segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - if d.fst != nil { + rv := &DictionaryIterator{ + d: d, + } + itr, err := d.fst.Search(a, startKeyInclusive, endKeyExclusive) if err == nil { rv.itr = itr } else if err != vellum.ErrIteratorDone { rv.err = err } - } - return rv -} - -func (d *Dictionary) OnlyIterator(onlyTerms [][]byte, - includeCount bool) segment.DictionaryIterator { - - rv := &DictionaryIterator{ - d: d, - omitCount: !includeCount, - } - - var buf bytes.Buffer - builder, err := vellum.New(&buf, nil) - if err != nil { - rv.err = err return rv } - for _, term := range onlyTerms { - err = builder.Insert(term, 0) - if err != nil { - rv.err = err - return rv - } - } - err = builder.Close() - if err != nil { - rv.err = err - return rv - } - - onlyFST, err := vellum.Load(buf.Bytes()) - if err != nil { - rv.err = err - return rv - } - - itr, err := d.fst.Search(onlyFST, nil, nil) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - - return rv + return emptyDictionaryIterator } // DictionaryIterator is an iterator for term dictionary @@ -242,6 +135,8 @@ type DictionaryIterator struct { omitCount bool } +var emptyDictionaryIterator = &DictionaryIterator{} + // Next returns the next entry in the dictionary func (i *DictionaryIterator) Next() (*index.DictEntry, error) { if i.err != nil && i.err != vellum.ErrIteratorDone { diff --git a/vendor/github.com/blevesearch/zap/v14/docvalues.go b/vendor/github.com/blevesearch/zapx/v14/docvalues.go similarity index 94% rename from vendor/github.com/blevesearch/zap/v14/docvalues.go rename to vendor/github.com/blevesearch/zapx/v14/docvalues.go index 793797bd8a..a530aa5ad9 100644 --- a/vendor/github.com/blevesearch/zap/v14/docvalues.go +++ b/vendor/github.com/blevesearch/zapx/v14/docvalues.go @@ -22,9 +22,8 @@ import ( "reflect" "sort" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/blevesearch/bleve/size" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" "github.com/golang/snappy" ) @@ -53,9 +52,9 @@ type docValueReader struct { } func (di *docValueReader) size() int { - return reflectStaticSizedocValueReader + size.SizeOfPtr + + return reflectStaticSizedocValueReader + SizeOfPtr + len(di.field) + - len(di.chunkOffsets)*size.SizeOfUint64 + + len(di.chunkOffsets)*SizeOfUint64 + len(di.curChunkHeader)*reflectStaticSizeMetaData + len(di.curChunkData) } @@ -200,7 +199,7 @@ func (di *docValueReader) iterateAllDocValues(s *SegmentBase, visitor docNumTerm } func (di *docValueReader) visitDocValues(docNum uint64, - visitor index.DocumentFieldTermVisitor) error { + visitor index.DocValueVisitor) error { // binary search the term locations for the docNum start, end := di.getDocValueLocs(docNum) if start == math.MaxUint64 || end == math.MaxUint64 || start == end { @@ -246,10 +245,10 @@ func (di *docValueReader) getDocValueLocs(docNum uint64) (uint64, uint64) { return math.MaxUint64, math.MaxUint64 } -// VisitDocumentFieldTerms is an implementation of the -// DocumentFieldTermVisitable interface -func (s *SegmentBase) VisitDocumentFieldTerms(localDocNum uint64, fields []string, - visitor index.DocumentFieldTermVisitor, dvsIn segment.DocVisitState) ( +// VisitDocValues is an implementation of the +// DocValueVisitable interface +func (s *SegmentBase) VisitDocValues(localDocNum uint64, fields []string, + visitor index.DocValueVisitor, dvsIn segment.DocVisitState) ( segment.DocVisitState, error) { dvs, ok := dvsIn.(*docVisitState) if !ok || dvs == nil { diff --git a/vendor/github.com/blevesearch/zap/v15/enumerator.go b/vendor/github.com/blevesearch/zapx/v14/enumerator.go similarity index 99% rename from vendor/github.com/blevesearch/zap/v15/enumerator.go rename to vendor/github.com/blevesearch/zapx/v14/enumerator.go index bc5b7e6276..972a224165 100644 --- a/vendor/github.com/blevesearch/zap/v15/enumerator.go +++ b/vendor/github.com/blevesearch/zapx/v14/enumerator.go @@ -17,7 +17,7 @@ package zap import ( "bytes" - "github.com/couchbase/vellum" + "github.com/blevesearch/vellum" ) // enumerator provides an ordered traversal of multiple vellum diff --git a/vendor/github.com/blevesearch/zapx/v14/go.mod b/vendor/github.com/blevesearch/zapx/v14/go.mod new file mode 100644 index 0000000000..ddd807ab02 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v14/go.mod @@ -0,0 +1,13 @@ +module github.com/blevesearch/zapx/v14 + +go 1.12 + +require ( + github.com/RoaringBitmap/roaring v0.7.3 + github.com/blevesearch/bleve_index_api v1.0.1 + github.com/blevesearch/mmap-go v1.0.2 + github.com/blevesearch/scorch_segment_api/v2 v2.0.1 + github.com/blevesearch/vellum v1.0.5 + github.com/golang/snappy v0.0.1 + github.com/spf13/cobra v0.0.5 +) diff --git a/vendor/github.com/blevesearch/zapx/v14/go.sum b/vendor/github.com/blevesearch/zapx/v14/go.sum new file mode 100644 index 0000000000..b6351ec69e --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v14/go.sum @@ -0,0 +1,97 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= +github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= +github.com/RoaringBitmap/roaring v0.7.3 h1:RwirWpvFONt2EwHHEHhER7S4BHZkyj3qL5LXLlnQPZ4= +github.com/RoaringBitmap/roaring v0.7.3/go.mod h1:jdT9ykXwHFNdJbEtxePexlFYH9LXucApeS0/+/g+p1I= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= +github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/blevesearch/bleve_index_api v1.0.0/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= +github.com/blevesearch/bleve_index_api v1.0.1 h1:nx9++0hnyiGOHJwQQYfsUGzpRdEVE5LsylmmngQvaFk= +github.com/blevesearch/bleve_index_api v1.0.1/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= +github.com/blevesearch/mmap-go v1.0.2 h1:JtMHb+FgQCTTYIhtMvimw15dJwu1Y5lrZDMOFXVWPk0= +github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA= +github.com/blevesearch/scorch_segment_api/v2 v2.0.1 h1:fd+hPtZ8GsbqPK1HslGp7Vhoik4arZteA/IsCEgOisw= +github.com/blevesearch/scorch_segment_api/v2 v2.0.1/go.mod h1:lq7yK2jQy1yQjtjTfU931aVqz7pYxEudHaDwOt1tXfU= +github.com/blevesearch/vellum v1.0.5 h1:L5dJ7hKauRVbuH7I8uqLeSK92CPPY6FfrbAmLhAug8A= +github.com/blevesearch/vellum v1.0.5/go.mod h1:atE0EH3fvk43zzS7t1YNdNC7DbmcC3uz+eMD5xZ2OyQ= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dvyukov/go-fuzz v0.0.0-20210429054444-fca39067bc72/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= +github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= +github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= +github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= +github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200928182047-19e03678916f/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/vendor/github.com/blevesearch/zap/v14/intDecoder.go b/vendor/github.com/blevesearch/zapx/v14/intDecoder.go similarity index 93% rename from vendor/github.com/blevesearch/zap/v14/intDecoder.go rename to vendor/github.com/blevesearch/zapx/v14/intDecoder.go index 3baa0c2a64..1b839e1c62 100644 --- a/vendor/github.com/blevesearch/zap/v14/intDecoder.go +++ b/vendor/github.com/blevesearch/zapx/v14/intDecoder.go @@ -17,8 +17,6 @@ package zap import ( "encoding/binary" "fmt" - - "github.com/blevesearch/bleve/index/scorch/segment" ) type chunkedIntDecoder struct { @@ -27,7 +25,7 @@ type chunkedIntDecoder struct { chunkOffsets []uint64 curChunkBytes []byte data []byte - r *segment.MemUvarintReader + r *memUvarintReader } // newChunkedIntDecoder expects an optional or reset chunkedIntDecoder for better reuse. @@ -63,7 +61,7 @@ func newChunkedIntDecoder(buf []byte, offset uint64, rv *chunkedIntDecoder) *chu func (d *chunkedIntDecoder) loadChunk(chunk int) error { if d.startOffset == termNotEncoded { - d.r = segment.NewMemUvarintReader([]byte(nil)) + d.r = newMemUvarintReader([]byte(nil)) return nil } @@ -78,7 +76,7 @@ func (d *chunkedIntDecoder) loadChunk(chunk int) error { end += e d.curChunkBytes = d.data[start:end] if d.r == nil { - d.r = segment.NewMemUvarintReader(d.curChunkBytes) + d.r = newMemUvarintReader(d.curChunkBytes) } else { d.r.Reset(d.curChunkBytes) } diff --git a/vendor/github.com/blevesearch/zap/v14/intcoder.go b/vendor/github.com/blevesearch/zapx/v14/intcoder.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v14/intcoder.go rename to vendor/github.com/blevesearch/zapx/v14/intcoder.go diff --git a/vendor/github.com/blevesearch/zapx/v14/memuvarint.go b/vendor/github.com/blevesearch/zapx/v14/memuvarint.go new file mode 100644 index 0000000000..0c10c83a4e --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v14/memuvarint.go @@ -0,0 +1,94 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "fmt" +) + +type memUvarintReader struct { + C int // index of next byte to read from S + S []byte +} + +func newMemUvarintReader(s []byte) *memUvarintReader { + return &memUvarintReader{S: s} +} + +// Len returns the number of unread bytes. +func (r *memUvarintReader) Len() int { + n := len(r.S) - r.C + if n < 0 { + return 0 + } + return n +} + +// ReadUvarint reads an encoded uint64. The original code this was +// based on is at encoding/binary/ReadUvarint(). +func (r *memUvarintReader) ReadUvarint() (uint64, error) { + var x uint64 + var s uint + var C = r.C + var S = r.S + + for { + b := S[C] + C++ + + if b < 0x80 { + r.C = C + + // why 63? The original code had an 'i += 1' loop var and + // checked for i > 9 || i == 9 ...; but, we no longer + // check for the i var, but instead check here for s, + // which is incremented by 7. So, 7*9 == 63. + // + // why the "extra" >= check? The normal case is that s < + // 63, so we check this single >= guard first so that we + // hit the normal, nil-error return pathway sooner. + if s >= 63 && (s > 63 || s == 63 && b > 1) { + return 0, fmt.Errorf("memUvarintReader overflow") + } + + return x | uint64(b)< uint32 */ sizeOfUints := 16 - sizeInBytes := (len(s.path) + size.SizeOfString) + sizeOfUints + sizeInBytes := (len(s.path) + SizeOfString) + sizeOfUints // mutex, refs -> int64 sizeInBytes += 16 @@ -246,7 +245,7 @@ func (s *SegmentBase) loadFields() error { func (s *SegmentBase) Dictionary(field string) (segment.TermDictionary, error) { dict, err := s.dictionary(field) if err == nil && dict == nil { - return &segment.EmptyDictionary{}, nil + return emptyDictionary, nil } return dict, err } @@ -282,7 +281,6 @@ func (sb *SegmentBase) dictionary(field string) (rv *Dictionary, err error) { if err != nil { return nil, fmt.Errorf("dictionary field %s vellum reader err: %v", field, err) } - } } @@ -304,16 +302,16 @@ var visitDocumentCtxPool = sync.Pool{ }, } -// VisitDocument invokes the DocFieldValueVistor for each stored field +// VisitStoredFields invokes the StoredFieldValueVisitor for each stored field // for the specified doc number -func (s *SegmentBase) VisitDocument(num uint64, visitor segment.DocumentFieldValueVisitor) error { +func (s *SegmentBase) VisitStoredFields(num uint64, visitor segment.StoredFieldValueVisitor) error { vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) defer visitDocumentCtxPool.Put(vdc) - return s.visitDocument(vdc, num, visitor) + return s.visitStoredFields(vdc, num, visitor) } -func (s *SegmentBase) visitDocument(vdc *visitDocumentCtx, num uint64, - visitor segment.DocumentFieldValueVisitor) error { +func (s *SegmentBase) visitStoredFields(vdc *visitDocumentCtx, num uint64, + visitor segment.StoredFieldValueVisitor) error { // first make sure this is a valid number in this segment if num < s.numDocs { meta, compressed := s.getDocStoredMetaAndCompressed(num) diff --git a/vendor/github.com/blevesearch/zapx/v14/sizes.go b/vendor/github.com/blevesearch/zapx/v14/sizes.go new file mode 100644 index 0000000000..34166ea330 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v14/sizes.go @@ -0,0 +1,59 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "reflect" +) + +func init() { + var b bool + SizeOfBool = int(reflect.TypeOf(b).Size()) + var f32 float32 + SizeOfFloat32 = int(reflect.TypeOf(f32).Size()) + var f64 float64 + SizeOfFloat64 = int(reflect.TypeOf(f64).Size()) + var i int + SizeOfInt = int(reflect.TypeOf(i).Size()) + var m map[int]int + SizeOfMap = int(reflect.TypeOf(m).Size()) + var ptr *int + SizeOfPtr = int(reflect.TypeOf(ptr).Size()) + var slice []int + SizeOfSlice = int(reflect.TypeOf(slice).Size()) + var str string + SizeOfString = int(reflect.TypeOf(str).Size()) + var u8 uint8 + SizeOfUint8 = int(reflect.TypeOf(u8).Size()) + var u16 uint16 + SizeOfUint16 = int(reflect.TypeOf(u16).Size()) + var u32 uint32 + SizeOfUint32 = int(reflect.TypeOf(u32).Size()) + var u64 uint64 + SizeOfUint64 = int(reflect.TypeOf(u64).Size()) +} + +var SizeOfBool int +var SizeOfFloat32 int +var SizeOfFloat64 int +var SizeOfInt int +var SizeOfMap int +var SizeOfPtr int +var SizeOfSlice int +var SizeOfString int +var SizeOfUint8 int +var SizeOfUint16 int +var SizeOfUint32 int +var SizeOfUint64 int diff --git a/vendor/github.com/blevesearch/zap/v14/write.go b/vendor/github.com/blevesearch/zapx/v14/write.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v14/write.go rename to vendor/github.com/blevesearch/zapx/v14/write.go diff --git a/vendor/github.com/blevesearch/zap/v14/zap.md b/vendor/github.com/blevesearch/zapx/v14/zap.md similarity index 100% rename from vendor/github.com/blevesearch/zap/v14/zap.md rename to vendor/github.com/blevesearch/zapx/v14/zap.md diff --git a/vendor/github.com/blevesearch/zap/v15/.gitignore b/vendor/github.com/blevesearch/zapx/v15/.gitignore similarity index 100% rename from vendor/github.com/blevesearch/zap/v15/.gitignore rename to vendor/github.com/blevesearch/zapx/v15/.gitignore diff --git a/vendor/github.com/blevesearch/zapx/v15/.golangci.yml b/vendor/github.com/blevesearch/zapx/v15/.golangci.yml new file mode 100644 index 0000000000..f0f2f6067f --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v15/.golangci.yml @@ -0,0 +1,29 @@ +linters: + # please, do not use `enable-all`: it's deprecated and will be removed soon. + # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint + disable-all: true + enable: + - bodyclose + - deadcode + - depguard + - dupl + - errcheck + - gofmt + - goimports + - goprintffuncname + - gosec + - gosimple + - govet + - ineffassign + - interfacer + - misspell + - nakedret + - nolintlint + - rowserrcheck + - staticcheck + - structcheck + - typecheck + - unused + - varcheck + - whitespace + diff --git a/vendor/github.com/blevesearch/zapx/v15/LICENSE b/vendor/github.com/blevesearch/zapx/v15/LICENSE new file mode 100644 index 0000000000..7a4a3ea242 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v15/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/vendor/github.com/blevesearch/zapx/v15/README.md b/vendor/github.com/blevesearch/zapx/v15/README.md new file mode 100644 index 0000000000..4cbf1a145b --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v15/README.md @@ -0,0 +1,163 @@ +# zapx file format + +The zapx module is fork of [zap](https://github.com/blevesearch/zap) module which maintains file format compatibility, but removes dependency on bleve, and instead depends only on the indepenent interface modules: + +- [bleve_index_api](https://github.com/blevesearch/scorch_segment_api) +- [scorch_segment_api](https://github.com/blevesearch/scorch_segment_api) + +Advanced ZAP File Format Documentation is [here](zap.md). + +The file is written in the reverse order that we typically access data. This helps us write in one pass since later sections of the file require file offsets of things we've already written. + +Current usage: + +- mmap the entire file +- crc-32 bytes and version are in fixed position at end of the file +- reading remainder of footer could be version specific +- remainder of footer gives us: + - 3 important offsets (docValue , fields index and stored data index) + - 2 important values (number of docs and chunk factor) +- field data is processed once and memoized onto the heap so that we never have to go back to disk for it +- access to stored data by doc number means first navigating to the stored data index, then accessing a fixed position offset into that slice, which gives us the actual address of the data. the first bytes of that section tell us the size of data so that we know where it ends. +- access to all other indexed data follows the following pattern: + - first know the field name -> convert to id + - next navigate to term dictionary for that field + - some operations stop here and do dictionary ops + - next use dictionary to navigate to posting list for a specific term + - walk posting list + - if necessary, walk posting details as we go + - if location info is desired, consult location bitmap to see if it is there + +## stored fields section + +- for each document + - preparation phase: + - produce a slice of metadata bytes and data bytes + - produce these slices in field id order + - field value is appended to the data slice + - metadata slice is varint encoded with the following values for each field value + - field id (uint16) + - field type (byte) + - field value start offset in uncompressed data slice (uint64) + - field value length (uint64) + - field number of array positions (uint64) + - one additional value for each array position (uint64) + - compress the data slice using snappy + - file writing phase: + - remember the start offset for this document + - write out meta data length (varint uint64) + - write out compressed data length (varint uint64) + - write out the metadata bytes + - write out the compressed data bytes + +## stored fields idx + +- for each document + - write start offset (remembered from previous section) of stored data (big endian uint64) + +With this index and a known document number, we have direct access to all the stored field data. + +## posting details (freq/norm) section + +- for each posting list + - produce a slice containing multiple consecutive chunks (each chunk is varint stream) + - produce a slice remembering offsets of where each chunk starts + - preparation phase: + - for each hit in the posting list + - if this hit is in next chunk close out encoding of last chunk and record offset start of next + - encode term frequency (uint64) + - encode norm factor (float32) + - file writing phase: + - remember start position for this posting list details + - write out number of chunks that follow (varint uint64) + - write out length of each chunk (each a varint uint64) + - write out the byte slice containing all the chunk data + +If you know the doc number you're interested in, this format lets you jump to the correct chunk (docNum/chunkFactor) directly and then seek within that chunk until you find it. + +## posting details (location) section + +- for each posting list + - produce a slice containing multiple consecutive chunks (each chunk is varint stream) + - produce a slice remembering offsets of where each chunk starts + - preparation phase: + - for each hit in the posting list + - if this hit is in next chunk close out encoding of last chunk and record offset start of next + - encode field (uint16) + - encode field pos (uint64) + - encode field start (uint64) + - encode field end (uint64) + - encode number of array positions to follow (uint64) + - encode each array position (each uint64) + - file writing phase: + - remember start position for this posting list details + - write out number of chunks that follow (varint uint64) + - write out length of each chunk (each a varint uint64) + - write out the byte slice containing all the chunk data + +If you know the doc number you're interested in, this format lets you jump to the correct chunk (docNum/chunkFactor) directly and then seek within that chunk until you find it. + +## postings list section + +- for each posting list + - preparation phase: + - encode roaring bitmap posting list to bytes (so we know the length) + - file writing phase: + - remember the start position for this posting list + - write freq/norm details offset (remembered from previous, as varint uint64) + - write location details offset (remembered from previous, as varint uint64) + - write length of encoded roaring bitmap + - write the serialized roaring bitmap data + +## dictionary + +- for each field + - preparation phase: + - encode vellum FST with dictionary data pointing to file offset of posting list (remembered from previous) + - file writing phase: + - remember the start position of this persistDictionary + - write length of vellum data (varint uint64) + - write out vellum data + +## fields section + +- for each field + - file writing phase: + - remember start offset for each field + - write dictionary address (remembered from previous) (varint uint64) + - write length of field name (varint uint64) + - write field name bytes + +## fields idx + +- for each field + - file writing phase: + - write big endian uint64 of start offset for each field + +NOTE: currently we don't know or record the length of this fields index. Instead we rely on the fact that we know it immediately precedes a footer of known size. + +## fields DocValue + +- for each field + - preparation phase: + - produce a slice containing multiple consecutive chunks, where each chunk is composed of a meta section followed by compressed columnar field data + - produce a slice remembering the length of each chunk + - file writing phase: + - remember the start position of this first field DocValue offset in the footer + - write out number of chunks that follow (varint uint64) + - write out length of each chunk (each a varint uint64) + - write out the byte slice containing all the chunk data + +NOTE: currently the meta header inside each chunk gives clue to the location offsets and size of the data pertaining to a given docID and any +read operation leverage that meta information to extract the document specific data from the file. + +## footer + +- file writing phase + - write number of docs (big endian uint64) + - write stored field index location (big endian uint64) + - write field index location (big endian uint64) + - write field docValue location (big endian uint64) + - write out chunk factor (big endian uint32) + - write out version (big endian uint32) + - write out file CRC of everything preceding this (big endian uint32) diff --git a/vendor/github.com/blevesearch/zap/v15/build.go b/vendor/github.com/blevesearch/zapx/v15/build.go similarity index 79% rename from vendor/github.com/blevesearch/zap/v15/build.go rename to vendor/github.com/blevesearch/zapx/v15/build.go index 1b7318376e..5db1d9ee24 100644 --- a/vendor/github.com/blevesearch/zap/v15/build.go +++ b/vendor/github.com/blevesearch/zapx/v15/build.go @@ -16,10 +16,12 @@ package zap import ( "bufio" + "fmt" + "io" "math" "os" - "github.com/couchbase/vellum" + "github.com/blevesearch/vellum" ) const Version uint32 = 15 @@ -32,6 +34,16 @@ func (sb *SegmentBase) Persist(path string) error { return PersistSegmentBase(sb, path) } +// WriteTo is an implementation of io.WriterTo interface. +func (sb *SegmentBase) WriteTo(w io.Writer) (int64, error) { + if w == nil { + return 0, fmt.Errorf("invalid writer found") + } + + n, err := persistSegmentBaseToWriter(sb, w) + return int64(n), err +} + // PersistSegmentBase persists SegmentBase in the zap file format. func PersistSegmentBase(sb *SegmentBase, path string) error { flag := os.O_RDWR | os.O_CREATE @@ -46,22 +58,7 @@ func PersistSegmentBase(sb *SegmentBase, path string) error { _ = os.Remove(path) } - br := bufio.NewWriter(f) - - _, err = br.Write(sb.mem) - if err != nil { - cleanup() - return err - } - - err = persistFooter(sb.numDocs, sb.storedIndexOffset, sb.fieldsIndexOffset, sb.docValueOffset, - sb.chunkMode, sb.memCRC, br) - if err != nil { - cleanup() - return err - } - - err = br.Flush() + _, err = persistSegmentBaseToWriter(sb, f) if err != nil { cleanup() return err @@ -79,7 +76,40 @@ func PersistSegmentBase(sb *SegmentBase, path string) error { return err } - return nil + return err +} + +type bufWriter struct { + w *bufio.Writer + n int +} + +func (br *bufWriter) Write(in []byte) (int, error) { + n, err := br.w.Write(in) + br.n += n + return n, err +} + +func persistSegmentBaseToWriter(sb *SegmentBase, w io.Writer) (int, error) { + br := &bufWriter{w: bufio.NewWriter(w)} + + _, err := br.Write(sb.mem) + if err != nil { + return 0, err + } + + err = persistFooter(sb.numDocs, sb.storedIndexOffset, sb.fieldsIndexOffset, + sb.docValueOffset, sb.chunkMode, sb.memCRC, br) + if err != nil { + return 0, err + } + + err = br.w.Flush() + if err != nil { + return 0, err + } + + return br.n, nil } func persistStoredFieldValues(fieldID int, diff --git a/vendor/github.com/blevesearch/zap/v15/chunk.go b/vendor/github.com/blevesearch/zapx/v15/chunk.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v15/chunk.go rename to vendor/github.com/blevesearch/zapx/v15/chunk.go diff --git a/vendor/github.com/blevesearch/zap/v15/contentcoder.go b/vendor/github.com/blevesearch/zapx/v15/contentcoder.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v15/contentcoder.go rename to vendor/github.com/blevesearch/zapx/v15/contentcoder.go diff --git a/vendor/github.com/blevesearch/zapx/v15/count.go b/vendor/github.com/blevesearch/zapx/v15/count.go new file mode 100644 index 0000000000..b6135359fb --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v15/count.go @@ -0,0 +1,61 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "hash/crc32" + "io" + + segment "github.com/blevesearch/scorch_segment_api/v2" +) + +// CountHashWriter is a wrapper around a Writer which counts the number of +// bytes which have been written and computes a crc32 hash +type CountHashWriter struct { + w io.Writer + crc uint32 + n int + s segment.StatsReporter +} + +// NewCountHashWriter returns a CountHashWriter which wraps the provided Writer +func NewCountHashWriter(w io.Writer) *CountHashWriter { + return &CountHashWriter{w: w} +} + +func NewCountHashWriterWithStatsReporter(w io.Writer, s segment.StatsReporter) *CountHashWriter { + return &CountHashWriter{w: w, s: s} +} + +// Write writes the provided bytes to the wrapped writer and counts the bytes +func (c *CountHashWriter) Write(b []byte) (int, error) { + n, err := c.w.Write(b) + c.crc = crc32.Update(c.crc, crc32.IEEETable, b[:n]) + c.n += n + if c.s != nil { + c.s.ReportBytesWritten(uint64(n)) + } + return n, err +} + +// Count returns the number of bytes written +func (c *CountHashWriter) Count() int { + return c.n +} + +// Sum32 returns the CRC-32 hash of the content written to this writer +func (c *CountHashWriter) Sum32() uint32 { + return c.crc +} diff --git a/vendor/github.com/blevesearch/zapx/v15/dict.go b/vendor/github.com/blevesearch/zapx/v15/dict.go new file mode 100644 index 0000000000..e30bf2420d --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v15/dict.go @@ -0,0 +1,158 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "fmt" + + "github.com/RoaringBitmap/roaring" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" + "github.com/blevesearch/vellum" +) + +// Dictionary is the zap representation of the term dictionary +type Dictionary struct { + sb *SegmentBase + field string + fieldID uint16 + fst *vellum.FST + fstReader *vellum.Reader +} + +// represents an immutable, empty dictionary +var emptyDictionary = &Dictionary{} + +// PostingsList returns the postings list for the specified term +func (d *Dictionary) PostingsList(term []byte, except *roaring.Bitmap, + prealloc segment.PostingsList) (segment.PostingsList, error) { + var preallocPL *PostingsList + pl, ok := prealloc.(*PostingsList) + if ok && pl != nil { + preallocPL = pl + } + return d.postingsList(term, except, preallocPL) +} + +func (d *Dictionary) postingsList(term []byte, except *roaring.Bitmap, rv *PostingsList) (*PostingsList, error) { + if d.fstReader == nil { + if rv == nil || rv == emptyPostingsList { + return emptyPostingsList, nil + } + return d.postingsListInit(rv, except), nil + } + + postingsOffset, exists, err := d.fstReader.Get(term) + if err != nil { + return nil, fmt.Errorf("vellum err: %v", err) + } + if !exists { + if rv == nil || rv == emptyPostingsList { + return emptyPostingsList, nil + } + return d.postingsListInit(rv, except), nil + } + + return d.postingsListFromOffset(postingsOffset, except, rv) +} + +func (d *Dictionary) postingsListFromOffset(postingsOffset uint64, except *roaring.Bitmap, rv *PostingsList) (*PostingsList, error) { + rv = d.postingsListInit(rv, except) + + err := rv.read(postingsOffset, d) + if err != nil { + return nil, err + } + + return rv, nil +} + +func (d *Dictionary) postingsListInit(rv *PostingsList, except *roaring.Bitmap) *PostingsList { + if rv == nil || rv == emptyPostingsList { + rv = &PostingsList{} + } else { + postings := rv.postings + if postings != nil { + postings.Clear() + } + + *rv = PostingsList{} // clear the struct + + rv.postings = postings + } + rv.sb = d.sb + rv.except = except + return rv +} + +func (d *Dictionary) Contains(key []byte) (bool, error) { + if d.fst != nil { + return d.fst.Contains(key) + } + return false, nil +} + +// AutomatonIterator returns an iterator which only visits terms +// having the the vellum automaton and start/end key range +func (d *Dictionary) AutomatonIterator(a segment.Automaton, + startKeyInclusive, endKeyExclusive []byte) segment.DictionaryIterator { + if d.fst != nil { + rv := &DictionaryIterator{ + d: d, + } + + itr, err := d.fst.Search(a, startKeyInclusive, endKeyExclusive) + if err == nil { + rv.itr = itr + } else if err != vellum.ErrIteratorDone { + rv.err = err + } + + return rv + } + return emptyDictionaryIterator +} + +// DictionaryIterator is an iterator for term dictionary +type DictionaryIterator struct { + d *Dictionary + itr vellum.Iterator + err error + tmp PostingsList + entry index.DictEntry + omitCount bool +} + +var emptyDictionaryIterator = &DictionaryIterator{} + +// Next returns the next entry in the dictionary +func (i *DictionaryIterator) Next() (*index.DictEntry, error) { + if i.err != nil && i.err != vellum.ErrIteratorDone { + return nil, i.err + } else if i.itr == nil || i.err == vellum.ErrIteratorDone { + return nil, nil + } + term, postingsOffset := i.itr.Current() + i.entry.Term = string(term) + if !i.omitCount { + i.err = i.tmp.read(postingsOffset, i.d) + if i.err != nil { + return nil, i.err + } + i.entry.Count = i.tmp.Count() + } + i.err = i.itr.Next() + return &i.entry, nil +} diff --git a/vendor/github.com/blevesearch/zap/v13/docvalues.go b/vendor/github.com/blevesearch/zapx/v15/docvalues.go similarity index 94% rename from vendor/github.com/blevesearch/zap/v13/docvalues.go rename to vendor/github.com/blevesearch/zapx/v15/docvalues.go index 793797bd8a..a530aa5ad9 100644 --- a/vendor/github.com/blevesearch/zap/v13/docvalues.go +++ b/vendor/github.com/blevesearch/zapx/v15/docvalues.go @@ -22,9 +22,8 @@ import ( "reflect" "sort" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/blevesearch/bleve/size" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" "github.com/golang/snappy" ) @@ -53,9 +52,9 @@ type docValueReader struct { } func (di *docValueReader) size() int { - return reflectStaticSizedocValueReader + size.SizeOfPtr + + return reflectStaticSizedocValueReader + SizeOfPtr + len(di.field) + - len(di.chunkOffsets)*size.SizeOfUint64 + + len(di.chunkOffsets)*SizeOfUint64 + len(di.curChunkHeader)*reflectStaticSizeMetaData + len(di.curChunkData) } @@ -200,7 +199,7 @@ func (di *docValueReader) iterateAllDocValues(s *SegmentBase, visitor docNumTerm } func (di *docValueReader) visitDocValues(docNum uint64, - visitor index.DocumentFieldTermVisitor) error { + visitor index.DocValueVisitor) error { // binary search the term locations for the docNum start, end := di.getDocValueLocs(docNum) if start == math.MaxUint64 || end == math.MaxUint64 || start == end { @@ -246,10 +245,10 @@ func (di *docValueReader) getDocValueLocs(docNum uint64) (uint64, uint64) { return math.MaxUint64, math.MaxUint64 } -// VisitDocumentFieldTerms is an implementation of the -// DocumentFieldTermVisitable interface -func (s *SegmentBase) VisitDocumentFieldTerms(localDocNum uint64, fields []string, - visitor index.DocumentFieldTermVisitor, dvsIn segment.DocVisitState) ( +// VisitDocValues is an implementation of the +// DocValueVisitable interface +func (s *SegmentBase) VisitDocValues(localDocNum uint64, fields []string, + visitor index.DocValueVisitor, dvsIn segment.DocVisitState) ( segment.DocVisitState, error) { dvs, ok := dvsIn.(*docVisitState) if !ok || dvs == nil { diff --git a/vendor/github.com/blevesearch/zap/v13/enumerator.go b/vendor/github.com/blevesearch/zapx/v15/enumerator.go similarity index 99% rename from vendor/github.com/blevesearch/zap/v13/enumerator.go rename to vendor/github.com/blevesearch/zapx/v15/enumerator.go index bc5b7e6276..972a224165 100644 --- a/vendor/github.com/blevesearch/zap/v13/enumerator.go +++ b/vendor/github.com/blevesearch/zapx/v15/enumerator.go @@ -17,7 +17,7 @@ package zap import ( "bytes" - "github.com/couchbase/vellum" + "github.com/blevesearch/vellum" ) // enumerator provides an ordered traversal of multiple vellum diff --git a/vendor/github.com/blevesearch/zapx/v15/go.mod b/vendor/github.com/blevesearch/zapx/v15/go.mod new file mode 100644 index 0000000000..6abf620011 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v15/go.mod @@ -0,0 +1,13 @@ +module github.com/blevesearch/zapx/v15 + +go 1.12 + +require ( + github.com/RoaringBitmap/roaring v0.7.3 + github.com/blevesearch/bleve_index_api v1.0.1 + github.com/blevesearch/mmap-go v1.0.2 + github.com/blevesearch/scorch_segment_api/v2 v2.0.1 + github.com/blevesearch/vellum v1.0.5 + github.com/golang/snappy v0.0.1 + github.com/spf13/cobra v0.0.5 +) diff --git a/vendor/github.com/blevesearch/zapx/v15/go.sum b/vendor/github.com/blevesearch/zapx/v15/go.sum new file mode 100644 index 0000000000..b6351ec69e --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v15/go.sum @@ -0,0 +1,97 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= +github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= +github.com/RoaringBitmap/roaring v0.7.3 h1:RwirWpvFONt2EwHHEHhER7S4BHZkyj3qL5LXLlnQPZ4= +github.com/RoaringBitmap/roaring v0.7.3/go.mod h1:jdT9ykXwHFNdJbEtxePexlFYH9LXucApeS0/+/g+p1I= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= +github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/blevesearch/bleve_index_api v1.0.0/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= +github.com/blevesearch/bleve_index_api v1.0.1 h1:nx9++0hnyiGOHJwQQYfsUGzpRdEVE5LsylmmngQvaFk= +github.com/blevesearch/bleve_index_api v1.0.1/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= +github.com/blevesearch/mmap-go v1.0.2 h1:JtMHb+FgQCTTYIhtMvimw15dJwu1Y5lrZDMOFXVWPk0= +github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA= +github.com/blevesearch/scorch_segment_api/v2 v2.0.1 h1:fd+hPtZ8GsbqPK1HslGp7Vhoik4arZteA/IsCEgOisw= +github.com/blevesearch/scorch_segment_api/v2 v2.0.1/go.mod h1:lq7yK2jQy1yQjtjTfU931aVqz7pYxEudHaDwOt1tXfU= +github.com/blevesearch/vellum v1.0.5 h1:L5dJ7hKauRVbuH7I8uqLeSK92CPPY6FfrbAmLhAug8A= +github.com/blevesearch/vellum v1.0.5/go.mod h1:atE0EH3fvk43zzS7t1YNdNC7DbmcC3uz+eMD5xZ2OyQ= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dvyukov/go-fuzz v0.0.0-20210429054444-fca39067bc72/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= +github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= +github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= +github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= +github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200928182047-19e03678916f/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/vendor/github.com/blevesearch/zap/v15/intDecoder.go b/vendor/github.com/blevesearch/zapx/v15/intDecoder.go similarity index 93% rename from vendor/github.com/blevesearch/zap/v15/intDecoder.go rename to vendor/github.com/blevesearch/zapx/v15/intDecoder.go index ea8021da47..2f777fc984 100644 --- a/vendor/github.com/blevesearch/zap/v15/intDecoder.go +++ b/vendor/github.com/blevesearch/zapx/v15/intDecoder.go @@ -17,8 +17,6 @@ package zap import ( "encoding/binary" "fmt" - - "github.com/blevesearch/bleve/index/scorch/segment" ) type chunkedIntDecoder struct { @@ -27,7 +25,7 @@ type chunkedIntDecoder struct { chunkOffsets []uint64 curChunkBytes []byte data []byte - r *segment.MemUvarintReader + r *memUvarintReader } // newChunkedIntDecoder expects an optional or reset chunkedIntDecoder for better reuse. @@ -63,7 +61,7 @@ func newChunkedIntDecoder(buf []byte, offset uint64, rv *chunkedIntDecoder) *chu func (d *chunkedIntDecoder) loadChunk(chunk int) error { if d.startOffset == termNotEncoded { - d.r = segment.NewMemUvarintReader([]byte(nil)) + d.r = newMemUvarintReader([]byte(nil)) return nil } @@ -78,7 +76,7 @@ func (d *chunkedIntDecoder) loadChunk(chunk int) error { end += e d.curChunkBytes = d.data[start:end] if d.r == nil { - d.r = segment.NewMemUvarintReader(d.curChunkBytes) + d.r = newMemUvarintReader(d.curChunkBytes) } else { d.r.Reset(d.curChunkBytes) } diff --git a/vendor/github.com/blevesearch/zap/v15/intcoder.go b/vendor/github.com/blevesearch/zapx/v15/intcoder.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v15/intcoder.go rename to vendor/github.com/blevesearch/zapx/v15/intcoder.go diff --git a/vendor/github.com/blevesearch/zapx/v15/memuvarint.go b/vendor/github.com/blevesearch/zapx/v15/memuvarint.go new file mode 100644 index 0000000000..0c10c83a4e --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v15/memuvarint.go @@ -0,0 +1,94 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "fmt" +) + +type memUvarintReader struct { + C int // index of next byte to read from S + S []byte +} + +func newMemUvarintReader(s []byte) *memUvarintReader { + return &memUvarintReader{S: s} +} + +// Len returns the number of unread bytes. +func (r *memUvarintReader) Len() int { + n := len(r.S) - r.C + if n < 0 { + return 0 + } + return n +} + +// ReadUvarint reads an encoded uint64. The original code this was +// based on is at encoding/binary/ReadUvarint(). +func (r *memUvarintReader) ReadUvarint() (uint64, error) { + var x uint64 + var s uint + var C = r.C + var S = r.S + + for { + b := S[C] + C++ + + if b < 0x80 { + r.C = C + + // why 63? The original code had an 'i += 1' loop var and + // checked for i > 9 || i == 9 ...; but, we no longer + // check for the i var, but instead check here for s, + // which is incremented by 7. So, 7*9 == 63. + // + // why the "extra" >= check? The normal case is that s < + // 63, so we check this single >= guard first so that we + // hit the normal, nil-error return pathway sooner. + if s >= 63 && (s > 63 || s == 63 && b > 1) { + return 0, fmt.Errorf("memUvarintReader overflow") + } + + return x | uint64(b)< 0), nextNorm) + if nextFreq > 0 { + err = tfEncoder.Add(hitNewDocNum, + encodeFreqHasLocs(nextFreq, len(locs) > 0), nextNorm) + } else { + err = tfEncoder.Add(hitNewDocNum, + encodeFreqHasLocs(nextFreq, len(locs) > 0)) + } if err != nil { return 0, 0, 0, nil, err } @@ -715,7 +717,7 @@ func mergeStoredAndRemap(segments []*SegmentBase, drops []*roaring.Bitmap, typs[i] = typs[i][:0] poss[i] = poss[i][:0] } - err := segment.visitDocument(vdc, docNum, func(field string, typ byte, value []byte, pos []uint64) bool { + err := segment.visitStoredFields(vdc, docNum, func(field string, typ byte, value []byte, pos []uint64) bool { fieldID := int(fieldsMap[field]) - 1 vals[fieldID] = append(vals[fieldID], value) typs[fieldID] = append(typs[fieldID], typ) diff --git a/vendor/github.com/blevesearch/zap/v15/new.go b/vendor/github.com/blevesearch/zapx/v15/new.go similarity index 88% rename from vendor/github.com/blevesearch/zap/v15/new.go rename to vendor/github.com/blevesearch/zapx/v15/new.go index c10a6a0685..362715d443 100644 --- a/vendor/github.com/blevesearch/zap/v15/new.go +++ b/vendor/github.com/blevesearch/zapx/v15/new.go @@ -22,11 +22,9 @@ import ( "sync" "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/couchbase/vellum" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" + "github.com/blevesearch/vellum" "github.com/golang/snappy" ) @@ -38,18 +36,17 @@ var NewSegmentBufferAvgBytesPerDocFactor float64 = 1.0 // on fields in a document being added to a new segment, by default it does // nothing. // This API is experimental and may be removed at any time. -var ValidateDocFields = func(field document.Field) error { +var ValidateDocFields = func(field index.Field) error { return nil } -// AnalysisResultsToSegmentBase produces an in-memory zap-encoded -// SegmentBase from analysis results -func (z *ZapPlugin) New(results []*index.AnalysisResult) ( +// New creates an in-memory zap-encoded SegmentBase from a set of Documents +func (z *ZapPlugin) New(results []index.Document) ( segment.Segment, uint64, error) { return z.newWithChunkMode(results, DefaultChunkMode) } -func (*ZapPlugin) newWithChunkMode(results []*index.AnalysisResult, +func (*ZapPlugin) newWithChunkMode(results []index.Document, chunkMode uint32) (segment.Segment, uint64, error) { s := interimPool.Get().(*interim) @@ -94,7 +91,7 @@ var interimPool = sync.Pool{New: func() interface{} { return &interim{} }} // interim holds temporary working data used while converting from // analysis results to a zap-encoded segment type interim struct { - results []*index.AnalysisResult + results []index.Document chunkMode uint32 @@ -228,12 +225,12 @@ func (s *interim) convert() (uint64, uint64, uint64, []uint64, error) { s.getOrDefineField("_id") // _id field is fieldID 0 for _, result := range s.results { - for _, field := range result.Document.CompositeFields { + result.VisitComposite(func(field index.CompositeField) { s.getOrDefineField(field.Name()) - } - for _, field := range result.Document.Fields { + }) + result.VisitFields(func(field index.Field) { s.getOrDefineField(field.Name()) - } + }) } sort.Strings(s.FieldsInv[1:]) // keep _id as first field @@ -309,10 +306,13 @@ func (s *interim) prepareDicts() { var totTFs int var totLocs int - visitField := func(fieldID uint16, tfs analysis.TokenFrequencies) { + visitField := func(field index.Field) { + fieldID := uint16(s.getOrDefineField(field.Name())) + dict := s.Dicts[fieldID] dictKeys := s.DictKeys[fieldID] + tfs := field.AnalyzedTokenFrequencies() for term, tf := range tfs { pidPlus1, exists := dict[term] if !exists { @@ -341,18 +341,12 @@ func (s *interim) prepareDicts() { for _, result := range s.results { // walk each composite field - for _, field := range result.Document.CompositeFields { - fieldID := uint16(s.getOrDefineField(field.Name())) - _, tf := field.Analyze() - visitField(fieldID, tf) - } + result.VisitComposite(func(field index.CompositeField) { + visitField(field) + }) // walk each field - for i, field := range result.Document.Fields { - fieldID := uint16(s.getOrDefineField(field.Name())) - tf := result.Analyzed[i] - visitField(fieldID, tf) - } + result.VisitFields(visitField) } numPostingsLists := pidNext @@ -410,7 +404,7 @@ func (s *interim) prepareDicts() { func (s *interim) processDocuments() { numFields := len(s.FieldsInv) reuseFieldLens := make([]int, numFields) - reuseFieldTFs := make([]analysis.TokenFrequencies, numFields) + reuseFieldTFs := make([]index.TokenFrequencies, numFields) for docNum, result := range s.results { for i := 0; i < numFields; i++ { // clear these for reuse @@ -424,34 +418,27 @@ func (s *interim) processDocuments() { } func (s *interim) processDocument(docNum uint64, - result *index.AnalysisResult, - fieldLens []int, fieldTFs []analysis.TokenFrequencies) { - visitField := func(fieldID uint16, fieldName string, - ln int, tf analysis.TokenFrequencies) { - fieldLens[fieldID] += ln + result index.Document, + fieldLens []int, fieldTFs []index.TokenFrequencies) { + visitField := func(field index.Field) { + fieldID := uint16(s.getOrDefineField(field.Name())) + fieldLens[fieldID] += field.AnalyzedLength() existingFreqs := fieldTFs[fieldID] if existingFreqs != nil { - existingFreqs.MergeAll(fieldName, tf) + existingFreqs.MergeAll(field.Name(), field.AnalyzedTokenFrequencies()) } else { - fieldTFs[fieldID] = tf + fieldTFs[fieldID] = field.AnalyzedTokenFrequencies() } } // walk each composite field - for _, field := range result.Document.CompositeFields { - fieldID := uint16(s.getOrDefineField(field.Name())) - ln, tf := field.Analyze() - visitField(fieldID, field.Name(), ln, tf) - } + result.VisitComposite(func(field index.CompositeField) { + visitField(field) + }) // walk each field - for i, field := range result.Document.Fields { - fieldID := uint16(s.getOrDefineField(field.Name())) - ln := result.Length[i] - tf := result.Analyzed[i] - visitField(fieldID, field.Name(), ln, tf) - } + result.VisitFields(visitField) // now that it's been rolled up into fieldTFs, walk that for fieldID, tfs := range fieldTFs { @@ -519,27 +506,29 @@ func (s *interim) writeStoredFields() ( delete(docStoredFields, fieldID) } - for _, field := range result.Document.Fields { + var validationErr error + result.VisitFields(func(field index.Field) { fieldID := uint16(s.getOrDefineField(field.Name())) - opts := field.Options() - - if opts.IsStored() { + if field.Options().IsStored() { isf := docStoredFields[fieldID] isf.vals = append(isf.vals, field.Value()) - isf.typs = append(isf.typs, encodeFieldType(field)) + isf.typs = append(isf.typs, field.EncodedFieldType()) isf.arrayposs = append(isf.arrayposs, field.ArrayPositions()) docStoredFields[fieldID] = isf } - if opts.IncludeDocValues() { + if field.Options().IncludeDocValues() { s.IncludeDocValues[fieldID] = true } err := ValidateDocFields(field) - if err != nil { - return 0, err + if err != nil && validationErr == nil { + validationErr = err } + }) + if validationErr != nil { + return 0, validationErr } var curr int @@ -667,9 +656,16 @@ func (s *interim) writeDicts() (fdvIndexOffset uint64, dictOffsets []uint64, err freqNorm := freqNorms[freqNormOffset] - err = tfEncoder.Add(docNum, - encodeFreqHasLocs(freqNorm.freq, freqNorm.numLocs > 0), - uint64(math.Float32bits(freqNorm.norm))) + // check if freq/norm is enabled + if freqNorm.freq > 0 { + err = tfEncoder.Add(docNum, + encodeFreqHasLocs(freqNorm.freq, freqNorm.numLocs > 0), + uint64(math.Float32bits(freqNorm.norm))) + } else { + // if disabled, then skip the norm part + err = tfEncoder.Add(docNum, + encodeFreqHasLocs(freqNorm.freq, freqNorm.numLocs > 0)) + } if err != nil { return 0, nil, err } @@ -817,25 +813,6 @@ func (s *interim) writeDicts() (fdvIndexOffset uint64, dictOffsets []uint64, err return fdvIndexOffset, dictOffsets, nil } -func encodeFieldType(f document.Field) byte { - fieldType := byte('x') - switch f.(type) { - case *document.TextField: - fieldType = 't' - case *document.NumericField: - fieldType = 'n' - case *document.DateTimeField: - fieldType = 'd' - case *document.BooleanField: - fieldType = 'b' - case *document.GeoPointField: - fieldType = 'g' - case *document.CompositeField: - fieldType = 'c' - } - return fieldType -} - // returns the total # of bytes needed to encode the given uint64's // into binary.PutUVarint() encoding func totalUvarintBytes(a, b, c, d, e uint64, more []uint64) (n int) { diff --git a/vendor/github.com/blevesearch/zapx/v15/plugin.go b/vendor/github.com/blevesearch/zapx/v15/plugin.go new file mode 100644 index 0000000000..f67297ec2f --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v15/plugin.go @@ -0,0 +1,27 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +// ZapPlugin implements the Plugin interface of +// the blevesearch/scorch_segment_api pkg +type ZapPlugin struct{} + +func (*ZapPlugin) Type() string { + return Type +} + +func (*ZapPlugin) Version() uint32 { + return Version +} diff --git a/vendor/github.com/blevesearch/zap/v15/posting.go b/vendor/github.com/blevesearch/zapx/v15/posting.go similarity index 95% rename from vendor/github.com/blevesearch/zap/v15/posting.go rename to vendor/github.com/blevesearch/zapx/v15/posting.go index 75faa5d702..b1d19e532d 100644 --- a/vendor/github.com/blevesearch/zap/v15/posting.go +++ b/vendor/github.com/blevesearch/zapx/v15/posting.go @@ -21,8 +21,7 @@ import ( "reflect" "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/blevesearch/bleve/size" + segment "github.com/blevesearch/scorch_segment_api/v2" ) var reflectStaticSizePostingsList int @@ -115,7 +114,7 @@ type PostingsList struct { var emptyPostingsList = &PostingsList{} func (p *PostingsList) Size() int { - sizeInBytes := reflectStaticSizePostingsList + size.SizeOfPtr + sizeInBytes := reflectStaticSizePostingsList + SizeOfPtr if p.except != nil { sizeInBytes += int(p.except.GetSizeInBytes()) @@ -322,7 +321,7 @@ type PostingsIterator struct { var emptyPostingsIterator = &PostingsIterator{} func (i *PostingsIterator) Size() int { - sizeInBytes := reflectStaticSizePostingsIterator + size.SizeOfPtr + + sizeInBytes := reflectStaticSizePostingsIterator + SizeOfPtr + i.next.Size() // account for freqNormReader, locReader if we start using this. for _, entry := range i.nextLocs { @@ -362,6 +361,9 @@ func (i *PostingsIterator) readFreqNormHasLocs() (uint64, uint64, bool, error) { } freq, hasLocs := decodeFreqHasLocs(freqHasLocs) + if freq == 0 { + return freq, 0, hasLocs, nil + } normBits, err := i.freqNormReader.readUvarint() if err != nil { @@ -381,9 +383,14 @@ func (i *PostingsIterator) skipFreqNormReadHasLocs() (bool, error) { return false, fmt.Errorf("error reading freqHasLocs: %v", err) } + freq, hasLocs := decodeFreqHasLocs(freqHasLocs) + if freq == 0 { + return hasLocs, nil + } + i.freqNormReader.SkipUvarint() // Skip normBits. - return freqHasLocs&0x01 != 0, nil // See decodeFreqHasLocs() / hasLocs. + return hasLocs, nil // See decodeFreqHasLocs() / hasLocs. } func encodeFreqHasLocs(freq uint64, hasLocs bool) uint64 { @@ -494,15 +501,17 @@ func (i *PostingsIterator) nextAtOrAfter(atOrAfter uint64) (segment.Posting, err // rv.freq >= "number of locs", since in a composite field, // some component fields might have their IncludeTermVector // flags disabled while other component fields are enabled - if cap(i.nextLocs) >= int(rv.freq) { - i.nextLocs = i.nextLocs[0:rv.freq] - } else { - i.nextLocs = make([]Location, rv.freq, rv.freq*2) + if rv.freq > 0 { + if cap(i.nextLocs) >= int(rv.freq) { + i.nextLocs = i.nextLocs[0:rv.freq] + } else { + i.nextLocs = make([]Location, rv.freq, rv.freq*2) + } + if cap(i.nextSegmentLocs) < int(rv.freq) { + i.nextSegmentLocs = make([]segment.Location, rv.freq, rv.freq*2) + } + rv.locs = i.nextSegmentLocs[:0] } - if cap(i.nextSegmentLocs) < int(rv.freq) { - i.nextSegmentLocs = make([]segment.Location, rv.freq, rv.freq*2) - } - rv.locs = i.nextSegmentLocs[:0] numLocsBytes, err := i.locReader.readUvarint() if err != nil { @@ -510,13 +519,21 @@ func (i *PostingsIterator) nextAtOrAfter(atOrAfter uint64) (segment.Posting, err } j := 0 + var nextLoc *Location startBytesRemaining := i.locReader.Len() // # bytes remaining in the locReader for startBytesRemaining-i.locReader.Len() < int(numLocsBytes) { - err := i.readLocation(&i.nextLocs[j]) + if len(i.nextLocs) > j { + nextLoc = &i.nextLocs[j] + } else { + nextLoc = &Location{} + } + + err := i.readLocation(nextLoc) if err != nil { return nil, err } - rv.locs = append(rv.locs, &i.nextLocs[j]) + + rv.locs = append(rv.locs, nextLoc) j++ } } @@ -644,7 +661,6 @@ func (i *PostingsIterator) nextBytes() ( // no deletions) where the all bitmap is the same as the actual bitmap func (i *PostingsIterator) nextDocNumAtOrAfterClean( atOrAfter uint64) (uint64, bool, error) { - if !i.includeFreqNorm { i.Actual.AdvanceIfNeeded(uint32(atOrAfter)) @@ -823,7 +839,7 @@ type Location struct { func (l *Location) Size() int { return reflectStaticSizeLocation + len(l.field) + - len(l.ap)*size.SizeOfUint64 + len(l.ap)*SizeOfUint64 } // Field returns the name of the field (useful in composite fields to know diff --git a/vendor/github.com/blevesearch/zap/v15/read.go b/vendor/github.com/blevesearch/zapx/v15/read.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v15/read.go rename to vendor/github.com/blevesearch/zapx/v15/read.go diff --git a/vendor/github.com/blevesearch/zap/v15/segment.go b/vendor/github.com/blevesearch/zapx/v15/segment.go similarity index 94% rename from vendor/github.com/blevesearch/zap/v15/segment.go rename to vendor/github.com/blevesearch/zapx/v15/segment.go index 2d158e86ea..bc29f3f4a3 100644 --- a/vendor/github.com/blevesearch/zap/v15/segment.go +++ b/vendor/github.com/blevesearch/zapx/v15/segment.go @@ -24,10 +24,9 @@ import ( "unsafe" "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/blevesearch/bleve/size" - "github.com/couchbase/vellum" mmap "github.com/blevesearch/mmap-go" + segment "github.com/blevesearch/scorch_segment_api/v2" + "github.com/blevesearch/vellum" "github.com/golang/snappy" ) @@ -117,18 +116,18 @@ func (sb *SegmentBase) updateSize() { // fieldsMap for k := range sb.fieldsMap { - sizeInBytes += (len(k) + size.SizeOfString) + size.SizeOfUint16 + sizeInBytes += (len(k) + SizeOfString) + SizeOfUint16 } // fieldsInv, dictLocs for _, entry := range sb.fieldsInv { - sizeInBytes += len(entry) + size.SizeOfString + sizeInBytes += len(entry) + SizeOfString } - sizeInBytes += len(sb.dictLocs) * size.SizeOfUint64 + sizeInBytes += len(sb.dictLocs) * SizeOfUint64 // fieldDvReaders for _, v := range sb.fieldDvReaders { - sizeInBytes += size.SizeOfUint16 + size.SizeOfPtr + sizeInBytes += SizeOfUint16 + SizeOfPtr if v != nil { sizeInBytes += v.size() } @@ -162,7 +161,7 @@ func (s *Segment) Size() int { // 4 /* size of crc -> uint32 */ sizeOfUints := 16 - sizeInBytes := (len(s.path) + size.SizeOfString) + sizeOfUints + sizeInBytes := (len(s.path) + SizeOfString) + sizeOfUints // mutex, refs -> int64 sizeInBytes += 16 @@ -246,7 +245,7 @@ func (s *SegmentBase) loadFields() error { func (s *SegmentBase) Dictionary(field string) (segment.TermDictionary, error) { dict, err := s.dictionary(field) if err == nil && dict == nil { - return &segment.EmptyDictionary{}, nil + return emptyDictionary, nil } return dict, err } @@ -282,7 +281,6 @@ func (sb *SegmentBase) dictionary(field string) (rv *Dictionary, err error) { if err != nil { return nil, fmt.Errorf("dictionary field %s vellum reader err: %v", field, err) } - } } @@ -304,16 +302,16 @@ var visitDocumentCtxPool = sync.Pool{ }, } -// VisitDocument invokes the DocFieldValueVistor for each stored field +// VisitStoredFields invokes the StoredFieldValueVisitor for each stored field // for the specified doc number -func (s *SegmentBase) VisitDocument(num uint64, visitor segment.DocumentFieldValueVisitor) error { +func (s *SegmentBase) VisitStoredFields(num uint64, visitor segment.StoredFieldValueVisitor) error { vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) defer visitDocumentCtxPool.Put(vdc) - return s.visitDocument(vdc, num, visitor) + return s.visitStoredFields(vdc, num, visitor) } -func (s *SegmentBase) visitDocument(vdc *visitDocumentCtx, num uint64, - visitor segment.DocumentFieldValueVisitor) error { +func (s *SegmentBase) visitStoredFields(vdc *visitDocumentCtx, num uint64, + visitor segment.StoredFieldValueVisitor) error { // first make sure this is a valid number in this segment if num < s.numDocs { meta, compressed := s.getDocStoredMetaAndCompressed(num) diff --git a/vendor/github.com/blevesearch/zapx/v15/sizes.go b/vendor/github.com/blevesearch/zapx/v15/sizes.go new file mode 100644 index 0000000000..34166ea330 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v15/sizes.go @@ -0,0 +1,59 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "reflect" +) + +func init() { + var b bool + SizeOfBool = int(reflect.TypeOf(b).Size()) + var f32 float32 + SizeOfFloat32 = int(reflect.TypeOf(f32).Size()) + var f64 float64 + SizeOfFloat64 = int(reflect.TypeOf(f64).Size()) + var i int + SizeOfInt = int(reflect.TypeOf(i).Size()) + var m map[int]int + SizeOfMap = int(reflect.TypeOf(m).Size()) + var ptr *int + SizeOfPtr = int(reflect.TypeOf(ptr).Size()) + var slice []int + SizeOfSlice = int(reflect.TypeOf(slice).Size()) + var str string + SizeOfString = int(reflect.TypeOf(str).Size()) + var u8 uint8 + SizeOfUint8 = int(reflect.TypeOf(u8).Size()) + var u16 uint16 + SizeOfUint16 = int(reflect.TypeOf(u16).Size()) + var u32 uint32 + SizeOfUint32 = int(reflect.TypeOf(u32).Size()) + var u64 uint64 + SizeOfUint64 = int(reflect.TypeOf(u64).Size()) +} + +var SizeOfBool int +var SizeOfFloat32 int +var SizeOfFloat64 int +var SizeOfInt int +var SizeOfMap int +var SizeOfPtr int +var SizeOfSlice int +var SizeOfString int +var SizeOfUint8 int +var SizeOfUint16 int +var SizeOfUint32 int +var SizeOfUint64 int diff --git a/vendor/github.com/blevesearch/zap/v15/write.go b/vendor/github.com/blevesearch/zapx/v15/write.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v15/write.go rename to vendor/github.com/blevesearch/zapx/v15/write.go diff --git a/vendor/github.com/blevesearch/zap/v15/zap.md b/vendor/github.com/blevesearch/zapx/v15/zap.md similarity index 100% rename from vendor/github.com/blevesearch/zap/v15/zap.md rename to vendor/github.com/blevesearch/zapx/v15/zap.md diff --git a/vendor/github.com/couchbase/vellum/.travis.yml b/vendor/github.com/couchbase/vellum/.travis.yml deleted file mode 100644 index 229edf2834..0000000000 --- a/vendor/github.com/couchbase/vellum/.travis.yml +++ /dev/null @@ -1,22 +0,0 @@ -sudo: false - -language: go - -go: - - "1.9.x" - - "1.10.x" - - "1.11.x" - -script: - - go get github.com/mattn/goveralls - - go get -u github.com/kisielk/errcheck - - go test -v $(go list ./... | grep -v vendor/) - - go test -race - - go vet - - errcheck - - go test -coverprofile=profile.out -covermode=count - - 'if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then goveralls -service=travis-ci -coverprofile=profile.out -repotoken $COVERALLS; fi' - -notifications: - email: - - marty.schoch@gmail.com diff --git a/vendor/github.com/willf/bitset/.gitignore b/vendor/github.com/willf/bitset/.gitignore deleted file mode 100644 index 5c204d28b0..0000000000 --- a/vendor/github.com/willf/bitset/.gitignore +++ /dev/null @@ -1,26 +0,0 @@ -# Compiled Object files, Static and Dynamic libs (Shared Objects) -*.o -*.a -*.so - -# Folders -_obj -_test - -# Architecture specific extensions/prefixes -*.[568vq] -[568vq].out - -*.cgo1.go -*.cgo2.c -_cgo_defun.c -_cgo_gotypes.go -_cgo_export.* - -_testmain.go - -*.exe -*.test -*.prof - -target diff --git a/vendor/github.com/willf/bitset/.travis.yml b/vendor/github.com/willf/bitset/.travis.yml deleted file mode 100644 index 094aa5ce07..0000000000 --- a/vendor/github.com/willf/bitset/.travis.yml +++ /dev/null @@ -1,37 +0,0 @@ -language: go - -sudo: false - -branches: - except: - - release - -branches: - only: - - master - - travis - -go: - - "1.11.x" - - tip - -matrix: - allow_failures: - - go: tip - -before_install: - - if [ -n "$GH_USER" ]; then git config --global github.user ${GH_USER}; fi; - - if [ -n "$GH_TOKEN" ]; then git config --global github.token ${GH_TOKEN}; fi; - - go get github.com/mattn/goveralls - -before_script: - - make deps - -script: - - make qa - -after_failure: - - cat ./target/test/report.xml - -after_success: - - if [ "$TRAVIS_GO_VERSION" = "1.11.1" ]; then $HOME/gopath/bin/goveralls -covermode=count -coverprofile=target/report/coverage.out -service=travis-ci; fi; diff --git a/vendor/github.com/willf/bitset/LICENSE b/vendor/github.com/willf/bitset/LICENSE deleted file mode 100644 index 59cab8a939..0000000000 --- a/vendor/github.com/willf/bitset/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2014 Will Fitzgerald. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/willf/bitset/README.md b/vendor/github.com/willf/bitset/README.md deleted file mode 100644 index 50338e71df..0000000000 --- a/vendor/github.com/willf/bitset/README.md +++ /dev/null @@ -1,94 +0,0 @@ -# bitset - -*Go language library to map between non-negative integers and boolean values* - -[![Test](https://github.com/willf/bitset/workflows/Test/badge.svg)](https://github.com/willf/bitset/actions?query=workflow%3ATest) -[![Master Coverage Status](https://coveralls.io/repos/willf/bitset/badge.svg?branch=master&service=github)](https://coveralls.io/github/willf/bitset?branch=master) -[![Go Report Card](https://goreportcard.com/badge/github.com/willf/bitset)](https://goreportcard.com/report/github.com/willf/bitset) -[![PkgGoDev](https://pkg.go.dev/badge/github.com/willf/bitset?tab=doc)](https://pkg.go.dev/github.com/willf/bitset?tab=doc) - - -## Description - -Package bitset implements bitsets, a mapping between non-negative integers and boolean values. -It should be more efficient than map[uint] bool. - -It provides methods for setting, clearing, flipping, and testing individual integers. - -But it also provides set intersection, union, difference, complement, and symmetric operations, as well as tests to check whether any, all, or no bits are set, and querying a bitset's current length and number of positive bits. - -BitSets are expanded to the size of the largest set bit; the memory allocation is approximately Max bits, where Max is the largest set bit. BitSets are never shrunk. On creation, a hint can be given for the number of bits that will be used. - -Many of the methods, including Set, Clear, and Flip, return a BitSet pointer, which allows for chaining. - -### Example use: - -```go -package main - -import ( - "fmt" - "math/rand" - - "github.com/willf/bitset" -) - -func main() { - fmt.Printf("Hello from BitSet!\n") - var b bitset.BitSet - // play some Go Fish - for i := 0; i < 100; i++ { - card1 := uint(rand.Intn(52)) - card2 := uint(rand.Intn(52)) - b.Set(card1) - if b.Test(card2) { - fmt.Println("Go Fish!") - } - b.Clear(card1) - } - - // Chaining - b.Set(10).Set(11) - - for i, e := b.NextSet(0); e; i, e = b.NextSet(i + 1) { - fmt.Println("The following bit is set:", i) - } - if b.Intersection(bitset.New(100).Set(10)).Count() == 1 { - fmt.Println("Intersection works.") - } else { - fmt.Println("Intersection doesn't work???") - } -} -``` - -As an alternative to BitSets, one should check out the 'big' package, which provides a (less set-theoretical) view of bitsets. - -Package documentation is at: https://pkg.go.dev/github.com/willf/bitset?tab=doc - -## Memory Usage - -The memory usage of a bitset using N bits is at least N/8 bytes. The number of bits in a bitset is at least as large as one plus the greatest bit index you have accessed. Thus it is possible to run out of memory while using a bitset. If you have lots of bits, you might prefer compressed bitsets, like the [Roaring bitmaps](http://roaringbitmap.org) and its [Go implementation](https://github.com/RoaringBitmap/roaring). - -## Implementation Note - -Go 1.9 introduced a native `math/bits` library. We provide backward compatibility to Go 1.7, which might be removed. - -It is possible that a later version will match the `math/bits` return signature for counts (which is `int`, rather than our library's `unit64`). If so, the version will be bumped. - -## Installation - -```bash -go get github.com/willf/bitset -``` - -## Contributing - -If you wish to contribute to this project, please branch and issue a pull request against master ("[GitHub Flow](https://guides.github.com/introduction/flow/)") - -## Running all tests - -Before committing the code, please check if it passes tests, has adequate coverage, etc. -```bash -go test -go test -cover -``` diff --git a/vendor/github.com/willf/bitset/azure-pipelines.yml b/vendor/github.com/willf/bitset/azure-pipelines.yml deleted file mode 100644 index f9b2959184..0000000000 --- a/vendor/github.com/willf/bitset/azure-pipelines.yml +++ /dev/null @@ -1,39 +0,0 @@ -# Go -# Build your Go project. -# Add steps that test, save build artifacts, deploy, and more: -# https://docs.microsoft.com/azure/devops/pipelines/languages/go - -trigger: -- master - -pool: - vmImage: 'Ubuntu-16.04' - -variables: - GOBIN: '$(GOPATH)/bin' # Go binaries path - GOROOT: '/usr/local/go1.11' # Go installation path - GOPATH: '$(system.defaultWorkingDirectory)/gopath' # Go workspace path - modulePath: '$(GOPATH)/src/github.com/$(build.repository.name)' # Path to the module's code - -steps: -- script: | - mkdir -p '$(GOBIN)' - mkdir -p '$(GOPATH)/pkg' - mkdir -p '$(modulePath)' - shopt -s extglob - shopt -s dotglob - mv !(gopath) '$(modulePath)' - echo '##vso[task.prependpath]$(GOBIN)' - echo '##vso[task.prependpath]$(GOROOT)/bin' - displayName: 'Set up the Go workspace' - -- script: | - go version - go get -v -t -d ./... - if [ -f Gopkg.toml ]; then - curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh - dep ensure - fi - go build -v . - workingDirectory: '$(modulePath)' - displayName: 'Get dependencies, then build' diff --git a/vendor/github.com/willf/bitset/bitset.go b/vendor/github.com/willf/bitset/bitset.go deleted file mode 100644 index 21e889da2e..0000000000 --- a/vendor/github.com/willf/bitset/bitset.go +++ /dev/null @@ -1,931 +0,0 @@ -/* -Package bitset implements bitsets, a mapping -between non-negative integers and boolean values. It should be more -efficient than map[uint] bool. - -It provides methods for setting, clearing, flipping, and testing -individual integers. - -But it also provides set intersection, union, difference, -complement, and symmetric operations, as well as tests to -check whether any, all, or no bits are set, and querying a -bitset's current length and number of positive bits. - -BitSets are expanded to the size of the largest set bit; the -memory allocation is approximately Max bits, where Max is -the largest set bit. BitSets are never shrunk. On creation, -a hint can be given for the number of bits that will be used. - -Many of the methods, including Set,Clear, and Flip, return -a BitSet pointer, which allows for chaining. - -Example use: - - import "bitset" - var b BitSet - b.Set(10).Set(11) - if b.Test(1000) { - b.Clear(1000) - } - if B.Intersection(bitset.New(100).Set(10)).Count() > 1 { - fmt.Println("Intersection works.") - } - -As an alternative to BitSets, one should check out the 'big' package, -which provides a (less set-theoretical) view of bitsets. - -*/ -package bitset - -import ( - "bufio" - "bytes" - "encoding/base64" - "encoding/binary" - "encoding/json" - "errors" - "fmt" - "io" - "strconv" -) - -// the wordSize of a bit set -const wordSize = uint(64) - -// log2WordSize is lg(wordSize) -const log2WordSize = uint(6) - -// allBits has every bit set -const allBits uint64 = 0xffffffffffffffff - -// default binary BigEndian -var binaryOrder binary.ByteOrder = binary.BigEndian - -// default json encoding base64.URLEncoding -var base64Encoding = base64.URLEncoding - -// Base64StdEncoding Marshal/Unmarshal BitSet with base64.StdEncoding(Default: base64.URLEncoding) -func Base64StdEncoding() { base64Encoding = base64.StdEncoding } - -// LittleEndian Marshal/Unmarshal Binary as Little Endian(Default: binary.BigEndian) -func LittleEndian() { binaryOrder = binary.LittleEndian } - -// A BitSet is a set of bits. The zero value of a BitSet is an empty set of length 0. -type BitSet struct { - length uint - set []uint64 -} - -// Error is used to distinguish errors (panics) generated in this package. -type Error string - -// safeSet will fixup b.set to be non-nil and return the field value -func (b *BitSet) safeSet() []uint64 { - if b.set == nil { - b.set = make([]uint64, wordsNeeded(0)) - } - return b.set -} - -// From is a constructor used to create a BitSet from an array of integers -func From(buf []uint64) *BitSet { - return &BitSet{uint(len(buf)) * 64, buf} -} - -// Bytes returns the bitset as array of integers -func (b *BitSet) Bytes() []uint64 { - return b.set -} - -// wordsNeeded calculates the number of words needed for i bits -func wordsNeeded(i uint) int { - if i > (Cap() - wordSize + 1) { - return int(Cap() >> log2WordSize) - } - return int((i + (wordSize - 1)) >> log2WordSize) -} - -// New creates a new BitSet with a hint that length bits will be required -func New(length uint) (bset *BitSet) { - defer func() { - if r := recover(); r != nil { - bset = &BitSet{ - 0, - make([]uint64, 0), - } - } - }() - - bset = &BitSet{ - length, - make([]uint64, wordsNeeded(length)), - } - - return bset -} - -// Cap returns the total possible capacity, or number of bits -func Cap() uint { - return ^uint(0) -} - -// Len returns the number of bits in the BitSet. -// Note the difference to method Count, see example. -func (b *BitSet) Len() uint { - return b.length -} - -// extendSetMaybe adds additional words to incorporate new bits if needed -func (b *BitSet) extendSetMaybe(i uint) { - if i >= b.length { // if we need more bits, make 'em - if i >= Cap() { - panic("You are exceeding the capacity") - } - nsize := wordsNeeded(i + 1) - if b.set == nil { - b.set = make([]uint64, nsize) - } else if cap(b.set) >= nsize { - b.set = b.set[:nsize] // fast resize - } else if len(b.set) < nsize { - newset := make([]uint64, nsize, 2*nsize) // increase capacity 2x - copy(newset, b.set) - b.set = newset - } - b.length = i + 1 - } -} - -// Test whether bit i is set. -func (b *BitSet) Test(i uint) bool { - if i >= b.length { - return false - } - return b.set[i>>log2WordSize]&(1<<(i&(wordSize-1))) != 0 -} - -// Set bit i to 1, the capacity of the bitset is automatically -// increased accordingly. -// If i>= Cap(), this function will panic. -// Warning: using a very large value for 'i' -// may lead to a memory shortage and a panic: the caller is responsible -// for providing sensible parameters in line with their memory capacity. -func (b *BitSet) Set(i uint) *BitSet { - b.extendSetMaybe(i) - b.set[i>>log2WordSize] |= 1 << (i & (wordSize - 1)) - return b -} - -// Clear bit i to 0 -func (b *BitSet) Clear(i uint) *BitSet { - if i >= b.length { - return b - } - b.set[i>>log2WordSize] &^= 1 << (i & (wordSize - 1)) - return b -} - -// SetTo sets bit i to value. -// If i>= Cap(), this function will panic. -// Warning: using a very large value for 'i' -// may lead to a memory shortage and a panic: the caller is responsible -// for providing sensible parameters in line with their memory capacity. -func (b *BitSet) SetTo(i uint, value bool) *BitSet { - if value { - return b.Set(i) - } - return b.Clear(i) -} - -// Flip bit at i. -// If i>= Cap(), this function will panic. -// Warning: using a very large value for 'i' -// may lead to a memory shortage and a panic: the caller is responsible -// for providing sensible parameters in line with their memory capacity. -func (b *BitSet) Flip(i uint) *BitSet { - if i >= b.length { - return b.Set(i) - } - b.set[i>>log2WordSize] ^= 1 << (i & (wordSize - 1)) - return b -} - -// Shrink shrinks BitSet so that the provided value is the last possible -// set value. It clears all bits > the provided index and reduces the size -// and length of the set. -// -// Note that the parameter value is not the new length in bits: it is the -// maximal value that can be stored in the bitset after the function call. -// The new length in bits is the parameter value + 1. Thus it is not possible -// to use this function to set the length to 0, the minimal value of the length -// after this function call is 1. -// -// A new slice is allocated to store the new bits, so you may see an increase in -// memory usage until the GC runs. Normally this should not be a problem, but if you -// have an extremely large BitSet its important to understand that the old BitSet will -// remain in memory until the GC frees it. -func (b *BitSet) Shrink(lastbitindex uint) *BitSet { - length := lastbitindex + 1 - idx := wordsNeeded(length) - if idx > len(b.set) { - return b - } - shrunk := make([]uint64, idx) - copy(shrunk, b.set[:idx]) - b.set = shrunk - b.length = length - b.set[idx-1] &= (allBits >> (uint64(64) - uint64(length&(wordSize-1)))) - return b -} - -// Compact shrinks BitSet to so that we preserve all set bits, while minimizing -// memory usage. Compact calls Shrink. -func (b *BitSet) Compact() *BitSet { - idx := len(b.set) - 1 - for ; idx >= 0 && b.set[idx] == 0; idx-- { - } - newlength := uint((idx + 1) << log2WordSize) - if newlength >= b.length { - return b // nothing to do - } - if newlength > 0 { - return b.Shrink(newlength - 1) - } - // We preserve one word - return b.Shrink(63) -} - -// InsertAt takes an index which indicates where a bit should be -// inserted. Then it shifts all the bits in the set to the left by 1, starting -// from the given index position, and sets the index position to 0. -// -// Depending on the size of your BitSet, and where you are inserting the new entry, -// this method could be extremely slow and in some cases might cause the entire BitSet -// to be recopied. -func (b *BitSet) InsertAt(idx uint) *BitSet { - insertAtElement := (idx >> log2WordSize) - - // if length of set is a multiple of wordSize we need to allocate more space first - if b.isLenExactMultiple() { - b.set = append(b.set, uint64(0)) - } - - var i uint - for i = uint(len(b.set) - 1); i > insertAtElement; i-- { - // all elements above the position where we want to insert can simply by shifted - b.set[i] <<= 1 - - // we take the most significant bit of the previous element and set it as - // the least significant bit of the current element - b.set[i] |= (b.set[i-1] & 0x8000000000000000) >> 63 - } - - // generate a mask to extract the data that we need to shift left - // within the element where we insert a bit - dataMask := ^(uint64(1)< 0x40000 { - buffer.WriteString("...") - break - } - buffer.WriteString(strconv.FormatInt(int64(i), 10)) - i, e = b.NextSet(i + 1) - if e { - buffer.WriteString(",") - } - } - buffer.WriteString("}") - return buffer.String() -} - -// DeleteAt deletes the bit at the given index position from -// within the bitset -// All the bits residing on the left of the deleted bit get -// shifted right by 1 -// The running time of this operation may potentially be -// relatively slow, O(length) -func (b *BitSet) DeleteAt(i uint) *BitSet { - // the index of the slice element where we'll delete a bit - deleteAtElement := i >> log2WordSize - - // generate a mask for the data that needs to be shifted right - // within that slice element that gets modified - dataMask := ^((uint64(1) << (i & (wordSize - 1))) - 1) - - // extract the data that we'll shift right from the slice element - data := b.set[deleteAtElement] & dataMask - - // set the masked area to 0 while leaving the rest as it is - b.set[deleteAtElement] &= ^dataMask - - // shift the previously extracted data to the right and then - // set it in the previously masked area - b.set[deleteAtElement] |= (data >> 1) & dataMask - - // loop over all the consecutive slice elements to copy each - // lowest bit into the highest position of the previous element, - // then shift the entire content to the right by 1 - for i := int(deleteAtElement) + 1; i < len(b.set); i++ { - b.set[i-1] |= (b.set[i] & 1) << 63 - b.set[i] >>= 1 - } - - b.length = b.length - 1 - - return b -} - -// NextSet returns the next bit set from the specified index, -// including possibly the current index -// along with an error code (true = valid, false = no set bit found) -// for i,e := v.NextSet(0); e; i,e = v.NextSet(i + 1) {...} -// -// Users concerned with performance may want to use NextSetMany to -// retrieve several values at once. -func (b *BitSet) NextSet(i uint) (uint, bool) { - x := int(i >> log2WordSize) - if x >= len(b.set) { - return 0, false - } - w := b.set[x] - w = w >> (i & (wordSize - 1)) - if w != 0 { - return i + trailingZeroes64(w), true - } - x = x + 1 - for x < len(b.set) { - if b.set[x] != 0 { - return uint(x)*wordSize + trailingZeroes64(b.set[x]), true - } - x = x + 1 - - } - return 0, false -} - -// NextSetMany returns many next bit sets from the specified index, -// including possibly the current index and up to cap(buffer). -// If the returned slice has len zero, then no more set bits were found -// -// buffer := make([]uint, 256) // this should be reused -// j := uint(0) -// j, buffer = bitmap.NextSetMany(j, buffer) -// for ; len(buffer) > 0; j, buffer = bitmap.NextSetMany(j,buffer) { -// for k := range buffer { -// do something with buffer[k] -// } -// j += 1 -// } -// -// -// It is possible to retrieve all set bits as follow: -// -// indices := make([]uint, bitmap.Count()) -// bitmap.NextSetMany(0, indices) -// -// However if bitmap.Count() is large, it might be preferable to -// use several calls to NextSetMany, for performance reasons. -func (b *BitSet) NextSetMany(i uint, buffer []uint) (uint, []uint) { - myanswer := buffer - capacity := cap(buffer) - x := int(i >> log2WordSize) - if x >= len(b.set) || capacity == 0 { - return 0, myanswer[:0] - } - skip := i & (wordSize - 1) - word := b.set[x] >> skip - myanswer = myanswer[:capacity] - size := int(0) - for word != 0 { - r := trailingZeroes64(word) - t := word & ((^word) + 1) - myanswer[size] = r + i - size++ - if size == capacity { - goto End - } - word = word ^ t - } - x++ - for idx, word := range b.set[x:] { - for word != 0 { - r := trailingZeroes64(word) - t := word & ((^word) + 1) - myanswer[size] = r + (uint(x+idx) << 6) - size++ - if size == capacity { - goto End - } - word = word ^ t - } - } -End: - if size > 0 { - return myanswer[size-1], myanswer[:size] - } - return 0, myanswer[:0] -} - -// NextClear returns the next clear bit from the specified index, -// including possibly the current index -// along with an error code (true = valid, false = no bit found i.e. all bits are set) -func (b *BitSet) NextClear(i uint) (uint, bool) { - x := int(i >> log2WordSize) - if x >= len(b.set) { - return 0, false - } - w := b.set[x] - w = w >> (i & (wordSize - 1)) - wA := allBits >> (i & (wordSize - 1)) - index := i + trailingZeroes64(^w) - if w != wA && index < b.length { - return index, true - } - x++ - for x < len(b.set) { - index = uint(x)*wordSize + trailingZeroes64(^b.set[x]) - if b.set[x] != allBits && index < b.length { - return index, true - } - x++ - } - return 0, false -} - -// ClearAll clears the entire BitSet -func (b *BitSet) ClearAll() *BitSet { - if b != nil && b.set != nil { - for i := range b.set { - b.set[i] = 0 - } - } - return b -} - -// wordCount returns the number of words used in a bit set -func (b *BitSet) wordCount() int { - return len(b.set) -} - -// Clone this BitSet -func (b *BitSet) Clone() *BitSet { - c := New(b.length) - if b.set != nil { // Clone should not modify current object - copy(c.set, b.set) - } - return c -} - -// Copy into a destination BitSet -// Returning the size of the destination BitSet -// like array copy -func (b *BitSet) Copy(c *BitSet) (count uint) { - if c == nil { - return - } - if b.set != nil { // Copy should not modify current object - copy(c.set, b.set) - } - count = c.length - if b.length < c.length { - count = b.length - } - return -} - -// Count (number of set bits). -// Also known as "popcount" or "popularity count". -func (b *BitSet) Count() uint { - if b != nil && b.set != nil { - return uint(popcntSlice(b.set)) - } - return 0 -} - -// Equal tests the equivalence of two BitSets. -// False if they are of different sizes, otherwise true -// only if all the same bits are set -func (b *BitSet) Equal(c *BitSet) bool { - if c == nil || b == nil { - return c == b - } - if b.length != c.length { - return false - } - if b.length == 0 { // if they have both length == 0, then could have nil set - return true - } - // testing for equality shoud not transform the bitset (no call to safeSet) - - for p, v := range b.set { - if c.set[p] != v { - return false - } - } - return true -} - -func panicIfNull(b *BitSet) { - if b == nil { - panic(Error("BitSet must not be null")) - } -} - -// Difference of base set and other set -// This is the BitSet equivalent of &^ (and not) -func (b *BitSet) Difference(compare *BitSet) (result *BitSet) { - panicIfNull(b) - panicIfNull(compare) - result = b.Clone() // clone b (in case b is bigger than compare) - l := int(compare.wordCount()) - if l > int(b.wordCount()) { - l = int(b.wordCount()) - } - for i := 0; i < l; i++ { - result.set[i] = b.set[i] &^ compare.set[i] - } - return -} - -// DifferenceCardinality computes the cardinality of the differnce -func (b *BitSet) DifferenceCardinality(compare *BitSet) uint { - panicIfNull(b) - panicIfNull(compare) - l := int(compare.wordCount()) - if l > int(b.wordCount()) { - l = int(b.wordCount()) - } - cnt := uint64(0) - cnt += popcntMaskSlice(b.set[:l], compare.set[:l]) - cnt += popcntSlice(b.set[l:]) - return uint(cnt) -} - -// InPlaceDifference computes the difference of base set and other set -// This is the BitSet equivalent of &^ (and not) -func (b *BitSet) InPlaceDifference(compare *BitSet) { - panicIfNull(b) - panicIfNull(compare) - l := int(compare.wordCount()) - if l > int(b.wordCount()) { - l = int(b.wordCount()) - } - for i := 0; i < l; i++ { - b.set[i] &^= compare.set[i] - } -} - -// Convenience function: return two bitsets ordered by -// increasing length. Note: neither can be nil -func sortByLength(a *BitSet, b *BitSet) (ap *BitSet, bp *BitSet) { - if a.length <= b.length { - ap, bp = a, b - } else { - ap, bp = b, a - } - return -} - -// Intersection of base set and other set -// This is the BitSet equivalent of & (and) -func (b *BitSet) Intersection(compare *BitSet) (result *BitSet) { - panicIfNull(b) - panicIfNull(compare) - b, compare = sortByLength(b, compare) - result = New(b.length) - for i, word := range b.set { - result.set[i] = word & compare.set[i] - } - return -} - -// IntersectionCardinality computes the cardinality of the union -func (b *BitSet) IntersectionCardinality(compare *BitSet) uint { - panicIfNull(b) - panicIfNull(compare) - b, compare = sortByLength(b, compare) - cnt := popcntAndSlice(b.set, compare.set) - return uint(cnt) -} - -// InPlaceIntersection destructively computes the intersection of -// base set and the compare set. -// This is the BitSet equivalent of & (and) -func (b *BitSet) InPlaceIntersection(compare *BitSet) { - panicIfNull(b) - panicIfNull(compare) - l := int(compare.wordCount()) - if l > int(b.wordCount()) { - l = int(b.wordCount()) - } - for i := 0; i < l; i++ { - b.set[i] &= compare.set[i] - } - for i := l; i < len(b.set); i++ { - b.set[i] = 0 - } - if compare.length > 0 { - b.extendSetMaybe(compare.length - 1) - } -} - -// Union of base set and other set -// This is the BitSet equivalent of | (or) -func (b *BitSet) Union(compare *BitSet) (result *BitSet) { - panicIfNull(b) - panicIfNull(compare) - b, compare = sortByLength(b, compare) - result = compare.Clone() - for i, word := range b.set { - result.set[i] = word | compare.set[i] - } - return -} - -// UnionCardinality computes the cardinality of the uniton of the base set -// and the compare set. -func (b *BitSet) UnionCardinality(compare *BitSet) uint { - panicIfNull(b) - panicIfNull(compare) - b, compare = sortByLength(b, compare) - cnt := popcntOrSlice(b.set, compare.set) - if len(compare.set) > len(b.set) { - cnt += popcntSlice(compare.set[len(b.set):]) - } - return uint(cnt) -} - -// InPlaceUnion creates the destructive union of base set and compare set. -// This is the BitSet equivalent of | (or). -func (b *BitSet) InPlaceUnion(compare *BitSet) { - panicIfNull(b) - panicIfNull(compare) - l := int(compare.wordCount()) - if l > int(b.wordCount()) { - l = int(b.wordCount()) - } - if compare.length > 0 { - b.extendSetMaybe(compare.length - 1) - } - for i := 0; i < l; i++ { - b.set[i] |= compare.set[i] - } - if len(compare.set) > l { - for i := l; i < len(compare.set); i++ { - b.set[i] = compare.set[i] - } - } -} - -// SymmetricDifference of base set and other set -// This is the BitSet equivalent of ^ (xor) -func (b *BitSet) SymmetricDifference(compare *BitSet) (result *BitSet) { - panicIfNull(b) - panicIfNull(compare) - b, compare = sortByLength(b, compare) - // compare is bigger, so clone it - result = compare.Clone() - for i, word := range b.set { - result.set[i] = word ^ compare.set[i] - } - return -} - -// SymmetricDifferenceCardinality computes the cardinality of the symmetric difference -func (b *BitSet) SymmetricDifferenceCardinality(compare *BitSet) uint { - panicIfNull(b) - panicIfNull(compare) - b, compare = sortByLength(b, compare) - cnt := popcntXorSlice(b.set, compare.set) - if len(compare.set) > len(b.set) { - cnt += popcntSlice(compare.set[len(b.set):]) - } - return uint(cnt) -} - -// InPlaceSymmetricDifference creates the destructive SymmetricDifference of base set and other set -// This is the BitSet equivalent of ^ (xor) -func (b *BitSet) InPlaceSymmetricDifference(compare *BitSet) { - panicIfNull(b) - panicIfNull(compare) - l := int(compare.wordCount()) - if l > int(b.wordCount()) { - l = int(b.wordCount()) - } - if compare.length > 0 { - b.extendSetMaybe(compare.length - 1) - } - for i := 0; i < l; i++ { - b.set[i] ^= compare.set[i] - } - if len(compare.set) > l { - for i := l; i < len(compare.set); i++ { - b.set[i] = compare.set[i] - } - } -} - -// Is the length an exact multiple of word sizes? -func (b *BitSet) isLenExactMultiple() bool { - return b.length%wordSize == 0 -} - -// Clean last word by setting unused bits to 0 -func (b *BitSet) cleanLastWord() { - if !b.isLenExactMultiple() { - b.set[len(b.set)-1] &= allBits >> (wordSize - b.length%wordSize) - } -} - -// Complement computes the (local) complement of a biset (up to length bits) -func (b *BitSet) Complement() (result *BitSet) { - panicIfNull(b) - result = New(b.length) - for i, word := range b.set { - result.set[i] = ^word - } - result.cleanLastWord() - return -} - -// All returns true if all bits are set, false otherwise. Returns true for -// empty sets. -func (b *BitSet) All() bool { - panicIfNull(b) - return b.Count() == b.length -} - -// None returns true if no bit is set, false otherwise. Returns true for -// empty sets. -func (b *BitSet) None() bool { - panicIfNull(b) - if b != nil && b.set != nil { - for _, word := range b.set { - if word > 0 { - return false - } - } - return true - } - return true -} - -// Any returns true if any bit is set, false otherwise -func (b *BitSet) Any() bool { - panicIfNull(b) - return !b.None() -} - -// IsSuperSet returns true if this is a superset of the other set -func (b *BitSet) IsSuperSet(other *BitSet) bool { - for i, e := other.NextSet(0); e; i, e = other.NextSet(i + 1) { - if !b.Test(i) { - return false - } - } - return true -} - -// IsStrictSuperSet returns true if this is a strict superset of the other set -func (b *BitSet) IsStrictSuperSet(other *BitSet) bool { - return b.Count() > other.Count() && b.IsSuperSet(other) -} - -// DumpAsBits dumps a bit set as a string of bits -func (b *BitSet) DumpAsBits() string { - if b.set == nil { - return "." - } - buffer := bytes.NewBufferString("") - i := len(b.set) - 1 - for ; i >= 0; i-- { - fmt.Fprintf(buffer, "%064b.", b.set[i]) - } - return buffer.String() -} - -// BinaryStorageSize returns the binary storage requirements -func (b *BitSet) BinaryStorageSize() int { - return binary.Size(uint64(0)) + binary.Size(b.set) -} - -// WriteTo writes a BitSet to a stream -func (b *BitSet) WriteTo(stream io.Writer) (int64, error) { - length := uint64(b.length) - - // Write length - err := binary.Write(stream, binaryOrder, length) - if err != nil { - return 0, err - } - - // Write set - err = binary.Write(stream, binaryOrder, b.set) - return int64(b.BinaryStorageSize()), err -} - -// ReadFrom reads a BitSet from a stream written using WriteTo -func (b *BitSet) ReadFrom(stream io.Reader) (int64, error) { - var length uint64 - - // Read length first - err := binary.Read(stream, binaryOrder, &length) - if err != nil { - return 0, err - } - newset := New(uint(length)) - - if uint64(newset.length) != length { - return 0, errors.New("unmarshalling error: type mismatch") - } - - // Read remaining bytes as set - err = binary.Read(stream, binaryOrder, newset.set) - if err != nil { - return 0, err - } - - *b = *newset - return int64(b.BinaryStorageSize()), nil -} - -// MarshalBinary encodes a BitSet into a binary form and returns the result. -func (b *BitSet) MarshalBinary() ([]byte, error) { - var buf bytes.Buffer - writer := bufio.NewWriter(&buf) - - _, err := b.WriteTo(writer) - if err != nil { - return []byte{}, err - } - - err = writer.Flush() - - return buf.Bytes(), err -} - -// UnmarshalBinary decodes the binary form generated by MarshalBinary. -func (b *BitSet) UnmarshalBinary(data []byte) error { - buf := bytes.NewReader(data) - reader := bufio.NewReader(buf) - - _, err := b.ReadFrom(reader) - - return err -} - -// MarshalJSON marshals a BitSet as a JSON structure -func (b *BitSet) MarshalJSON() ([]byte, error) { - buffer := bytes.NewBuffer(make([]byte, 0, b.BinaryStorageSize())) - _, err := b.WriteTo(buffer) - if err != nil { - return nil, err - } - - // URLEncode all bytes - return json.Marshal(base64Encoding.EncodeToString(buffer.Bytes())) -} - -// UnmarshalJSON unmarshals a BitSet from JSON created using MarshalJSON -func (b *BitSet) UnmarshalJSON(data []byte) error { - // Unmarshal as string - var s string - err := json.Unmarshal(data, &s) - if err != nil { - return err - } - - // URLDecode string - buf, err := base64Encoding.DecodeString(s) - if err != nil { - return err - } - - _, err = b.ReadFrom(bytes.NewReader(buf)) - return err -} diff --git a/vendor/github.com/willf/bitset/go.mod b/vendor/github.com/willf/bitset/go.mod deleted file mode 100644 index 583ecab78f..0000000000 --- a/vendor/github.com/willf/bitset/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module github.com/willf/bitset - -go 1.14 diff --git a/vendor/github.com/willf/bitset/popcnt.go b/vendor/github.com/willf/bitset/popcnt.go deleted file mode 100644 index 76577a8382..0000000000 --- a/vendor/github.com/willf/bitset/popcnt.go +++ /dev/null @@ -1,53 +0,0 @@ -package bitset - -// bit population count, take from -// https://code.google.com/p/go/issues/detail?id=4988#c11 -// credit: https://code.google.com/u/arnehormann/ -func popcount(x uint64) (n uint64) { - x -= (x >> 1) & 0x5555555555555555 - x = (x>>2)&0x3333333333333333 + x&0x3333333333333333 - x += x >> 4 - x &= 0x0f0f0f0f0f0f0f0f - x *= 0x0101010101010101 - return x >> 56 -} - -func popcntSliceGo(s []uint64) uint64 { - cnt := uint64(0) - for _, x := range s { - cnt += popcount(x) - } - return cnt -} - -func popcntMaskSliceGo(s, m []uint64) uint64 { - cnt := uint64(0) - for i := range s { - cnt += popcount(s[i] &^ m[i]) - } - return cnt -} - -func popcntAndSliceGo(s, m []uint64) uint64 { - cnt := uint64(0) - for i := range s { - cnt += popcount(s[i] & m[i]) - } - return cnt -} - -func popcntOrSliceGo(s, m []uint64) uint64 { - cnt := uint64(0) - for i := range s { - cnt += popcount(s[i] | m[i]) - } - return cnt -} - -func popcntXorSliceGo(s, m []uint64) uint64 { - cnt := uint64(0) - for i := range s { - cnt += popcount(s[i] ^ m[i]) - } - return cnt -} diff --git a/vendor/github.com/willf/bitset/popcnt_19.go b/vendor/github.com/willf/bitset/popcnt_19.go deleted file mode 100644 index fc8ff4f367..0000000000 --- a/vendor/github.com/willf/bitset/popcnt_19.go +++ /dev/null @@ -1,45 +0,0 @@ -// +build go1.9 - -package bitset - -import "math/bits" - -func popcntSlice(s []uint64) uint64 { - var cnt int - for _, x := range s { - cnt += bits.OnesCount64(x) - } - return uint64(cnt) -} - -func popcntMaskSlice(s, m []uint64) uint64 { - var cnt int - for i := range s { - cnt += bits.OnesCount64(s[i] &^ m[i]) - } - return uint64(cnt) -} - -func popcntAndSlice(s, m []uint64) uint64 { - var cnt int - for i := range s { - cnt += bits.OnesCount64(s[i] & m[i]) - } - return uint64(cnt) -} - -func popcntOrSlice(s, m []uint64) uint64 { - var cnt int - for i := range s { - cnt += bits.OnesCount64(s[i] | m[i]) - } - return uint64(cnt) -} - -func popcntXorSlice(s, m []uint64) uint64 { - var cnt int - for i := range s { - cnt += bits.OnesCount64(s[i] ^ m[i]) - } - return uint64(cnt) -} diff --git a/vendor/github.com/willf/bitset/popcnt_amd64.go b/vendor/github.com/willf/bitset/popcnt_amd64.go deleted file mode 100644 index 4cf64f24ad..0000000000 --- a/vendor/github.com/willf/bitset/popcnt_amd64.go +++ /dev/null @@ -1,68 +0,0 @@ -// +build !go1.9 -// +build amd64,!appengine - -package bitset - -// *** the following functions are defined in popcnt_amd64.s - -//go:noescape - -func hasAsm() bool - -// useAsm is a flag used to select the GO or ASM implementation of the popcnt function -var useAsm = hasAsm() - -//go:noescape - -func popcntSliceAsm(s []uint64) uint64 - -//go:noescape - -func popcntMaskSliceAsm(s, m []uint64) uint64 - -//go:noescape - -func popcntAndSliceAsm(s, m []uint64) uint64 - -//go:noescape - -func popcntOrSliceAsm(s, m []uint64) uint64 - -//go:noescape - -func popcntXorSliceAsm(s, m []uint64) uint64 - -func popcntSlice(s []uint64) uint64 { - if useAsm { - return popcntSliceAsm(s) - } - return popcntSliceGo(s) -} - -func popcntMaskSlice(s, m []uint64) uint64 { - if useAsm { - return popcntMaskSliceAsm(s, m) - } - return popcntMaskSliceGo(s, m) -} - -func popcntAndSlice(s, m []uint64) uint64 { - if useAsm { - return popcntAndSliceAsm(s, m) - } - return popcntAndSliceGo(s, m) -} - -func popcntOrSlice(s, m []uint64) uint64 { - if useAsm { - return popcntOrSliceAsm(s, m) - } - return popcntOrSliceGo(s, m) -} - -func popcntXorSlice(s, m []uint64) uint64 { - if useAsm { - return popcntXorSliceAsm(s, m) - } - return popcntXorSliceGo(s, m) -} diff --git a/vendor/github.com/willf/bitset/popcnt_amd64.s b/vendor/github.com/willf/bitset/popcnt_amd64.s deleted file mode 100644 index 666c0dcc17..0000000000 --- a/vendor/github.com/willf/bitset/popcnt_amd64.s +++ /dev/null @@ -1,104 +0,0 @@ -// +build !go1.9 -// +build amd64,!appengine - -TEXT ·hasAsm(SB),4,$0-1 -MOVQ $1, AX -CPUID -SHRQ $23, CX -ANDQ $1, CX -MOVB CX, ret+0(FP) -RET - -#define POPCNTQ_DX_DX BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0xd2 - -TEXT ·popcntSliceAsm(SB),4,$0-32 -XORQ AX, AX -MOVQ s+0(FP), SI -MOVQ s_len+8(FP), CX -TESTQ CX, CX -JZ popcntSliceEnd -popcntSliceLoop: -BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0x16 // POPCNTQ (SI), DX -ADDQ DX, AX -ADDQ $8, SI -LOOP popcntSliceLoop -popcntSliceEnd: -MOVQ AX, ret+24(FP) -RET - -TEXT ·popcntMaskSliceAsm(SB),4,$0-56 -XORQ AX, AX -MOVQ s+0(FP), SI -MOVQ s_len+8(FP), CX -TESTQ CX, CX -JZ popcntMaskSliceEnd -MOVQ m+24(FP), DI -popcntMaskSliceLoop: -MOVQ (DI), DX -NOTQ DX -ANDQ (SI), DX -POPCNTQ_DX_DX -ADDQ DX, AX -ADDQ $8, SI -ADDQ $8, DI -LOOP popcntMaskSliceLoop -popcntMaskSliceEnd: -MOVQ AX, ret+48(FP) -RET - -TEXT ·popcntAndSliceAsm(SB),4,$0-56 -XORQ AX, AX -MOVQ s+0(FP), SI -MOVQ s_len+8(FP), CX -TESTQ CX, CX -JZ popcntAndSliceEnd -MOVQ m+24(FP), DI -popcntAndSliceLoop: -MOVQ (DI), DX -ANDQ (SI), DX -POPCNTQ_DX_DX -ADDQ DX, AX -ADDQ $8, SI -ADDQ $8, DI -LOOP popcntAndSliceLoop -popcntAndSliceEnd: -MOVQ AX, ret+48(FP) -RET - -TEXT ·popcntOrSliceAsm(SB),4,$0-56 -XORQ AX, AX -MOVQ s+0(FP), SI -MOVQ s_len+8(FP), CX -TESTQ CX, CX -JZ popcntOrSliceEnd -MOVQ m+24(FP), DI -popcntOrSliceLoop: -MOVQ (DI), DX -ORQ (SI), DX -POPCNTQ_DX_DX -ADDQ DX, AX -ADDQ $8, SI -ADDQ $8, DI -LOOP popcntOrSliceLoop -popcntOrSliceEnd: -MOVQ AX, ret+48(FP) -RET - -TEXT ·popcntXorSliceAsm(SB),4,$0-56 -XORQ AX, AX -MOVQ s+0(FP), SI -MOVQ s_len+8(FP), CX -TESTQ CX, CX -JZ popcntXorSliceEnd -MOVQ m+24(FP), DI -popcntXorSliceLoop: -MOVQ (DI), DX -XORQ (SI), DX -POPCNTQ_DX_DX -ADDQ DX, AX -ADDQ $8, SI -ADDQ $8, DI -LOOP popcntXorSliceLoop -popcntXorSliceEnd: -MOVQ AX, ret+48(FP) -RET diff --git a/vendor/github.com/willf/bitset/popcnt_generic.go b/vendor/github.com/willf/bitset/popcnt_generic.go deleted file mode 100644 index 21e0ff7b4f..0000000000 --- a/vendor/github.com/willf/bitset/popcnt_generic.go +++ /dev/null @@ -1,24 +0,0 @@ -// +build !go1.9 -// +build !amd64 appengine - -package bitset - -func popcntSlice(s []uint64) uint64 { - return popcntSliceGo(s) -} - -func popcntMaskSlice(s, m []uint64) uint64 { - return popcntMaskSliceGo(s, m) -} - -func popcntAndSlice(s, m []uint64) uint64 { - return popcntAndSliceGo(s, m) -} - -func popcntOrSlice(s, m []uint64) uint64 { - return popcntOrSliceGo(s, m) -} - -func popcntXorSlice(s, m []uint64) uint64 { - return popcntXorSliceGo(s, m) -} diff --git a/vendor/github.com/willf/bitset/trailing_zeros_18.go b/vendor/github.com/willf/bitset/trailing_zeros_18.go deleted file mode 100644 index c52b61be9f..0000000000 --- a/vendor/github.com/willf/bitset/trailing_zeros_18.go +++ /dev/null @@ -1,14 +0,0 @@ -// +build !go1.9 - -package bitset - -var deBruijn = [...]byte{ - 0, 1, 56, 2, 57, 49, 28, 3, 61, 58, 42, 50, 38, 29, 17, 4, - 62, 47, 59, 36, 45, 43, 51, 22, 53, 39, 33, 30, 24, 18, 12, 5, - 63, 55, 48, 27, 60, 41, 37, 16, 46, 35, 44, 21, 52, 32, 23, 11, - 54, 26, 40, 15, 34, 20, 31, 10, 25, 14, 19, 9, 13, 8, 7, 6, -} - -func trailingZeroes64(v uint64) uint { - return uint(deBruijn[((v&-v)*0x03f79d71b4ca8b09)>>58]) -} diff --git a/vendor/github.com/willf/bitset/trailing_zeros_19.go b/vendor/github.com/willf/bitset/trailing_zeros_19.go deleted file mode 100644 index 36a988e714..0000000000 --- a/vendor/github.com/willf/bitset/trailing_zeros_19.go +++ /dev/null @@ -1,9 +0,0 @@ -// +build go1.9 - -package bitset - -import "math/bits" - -func trailingZeroes64(v uint64) uint { - return uint(bits.TrailingZeros64(v)) -} diff --git a/vendor/modules.txt b/vendor/modules.txt index dc753f549d..ddcdffece3 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -90,72 +90,75 @@ github.com/bits-and-blooms/bitset # github.com/blang/semver v3.5.1+incompatible ## explicit github.com/blang/semver -# github.com/blevesearch/bleve v1.0.14 +# github.com/blevesearch/bleve/v2 v2.1.0 ## explicit -github.com/blevesearch/bleve -github.com/blevesearch/bleve/analysis -github.com/blevesearch/bleve/analysis/analyzer/keyword -github.com/blevesearch/bleve/analysis/analyzer/standard -github.com/blevesearch/bleve/analysis/datetime/flexible -github.com/blevesearch/bleve/analysis/datetime/optional -github.com/blevesearch/bleve/analysis/lang/en -github.com/blevesearch/bleve/analysis/token/lowercase -github.com/blevesearch/bleve/analysis/token/porter -github.com/blevesearch/bleve/analysis/token/stop -github.com/blevesearch/bleve/analysis/tokenizer/single -github.com/blevesearch/bleve/analysis/tokenizer/unicode -github.com/blevesearch/bleve/document -github.com/blevesearch/bleve/geo -github.com/blevesearch/bleve/index -github.com/blevesearch/bleve/index/scorch -github.com/blevesearch/bleve/index/scorch/mergeplan -github.com/blevesearch/bleve/index/scorch/segment -github.com/blevesearch/bleve/index/store -github.com/blevesearch/bleve/index/store/boltdb -github.com/blevesearch/bleve/index/store/gtreap -github.com/blevesearch/bleve/index/upsidedown -github.com/blevesearch/bleve/mapping -github.com/blevesearch/bleve/numeric -github.com/blevesearch/bleve/registry -github.com/blevesearch/bleve/search -github.com/blevesearch/bleve/search/collector -github.com/blevesearch/bleve/search/facet -github.com/blevesearch/bleve/search/highlight -github.com/blevesearch/bleve/search/highlight/format/html -github.com/blevesearch/bleve/search/highlight/fragmenter/simple -github.com/blevesearch/bleve/search/highlight/highlighter/html -github.com/blevesearch/bleve/search/highlight/highlighter/simple -github.com/blevesearch/bleve/search/query -github.com/blevesearch/bleve/search/scorer -github.com/blevesearch/bleve/search/searcher -github.com/blevesearch/bleve/size +github.com/blevesearch/bleve/v2 +github.com/blevesearch/bleve/v2/analysis +github.com/blevesearch/bleve/v2/analysis/analyzer/keyword +github.com/blevesearch/bleve/v2/analysis/analyzer/standard +github.com/blevesearch/bleve/v2/analysis/datetime/flexible +github.com/blevesearch/bleve/v2/analysis/datetime/optional +github.com/blevesearch/bleve/v2/analysis/lang/en +github.com/blevesearch/bleve/v2/analysis/token/lowercase +github.com/blevesearch/bleve/v2/analysis/token/porter +github.com/blevesearch/bleve/v2/analysis/token/stop +github.com/blevesearch/bleve/v2/analysis/tokenizer/single +github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode +github.com/blevesearch/bleve/v2/document +github.com/blevesearch/bleve/v2/geo +github.com/blevesearch/bleve/v2/index/scorch +github.com/blevesearch/bleve/v2/index/scorch/mergeplan +github.com/blevesearch/bleve/v2/index/upsidedown +github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb +github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap +github.com/blevesearch/bleve/v2/mapping +github.com/blevesearch/bleve/v2/numeric +github.com/blevesearch/bleve/v2/registry +github.com/blevesearch/bleve/v2/search +github.com/blevesearch/bleve/v2/search/collector +github.com/blevesearch/bleve/v2/search/facet +github.com/blevesearch/bleve/v2/search/highlight +github.com/blevesearch/bleve/v2/search/highlight/format/html +github.com/blevesearch/bleve/v2/search/highlight/fragmenter/simple +github.com/blevesearch/bleve/v2/search/highlight/highlighter/html +github.com/blevesearch/bleve/v2/search/highlight/highlighter/simple +github.com/blevesearch/bleve/v2/search/query +github.com/blevesearch/bleve/v2/search/scorer +github.com/blevesearch/bleve/v2/search/searcher +github.com/blevesearch/bleve/v2/size +# github.com/blevesearch/bleve_index_api v1.0.1 +github.com/blevesearch/bleve_index_api # github.com/blevesearch/go-porterstemmer v1.0.3 github.com/blevesearch/go-porterstemmer # github.com/blevesearch/mmap-go v1.0.2 github.com/blevesearch/mmap-go +# github.com/blevesearch/scorch_segment_api/v2 v2.0.1 +github.com/blevesearch/scorch_segment_api/v2 # github.com/blevesearch/segment v0.9.0 github.com/blevesearch/segment # github.com/blevesearch/snowballstem v0.9.0 github.com/blevesearch/snowballstem github.com/blevesearch/snowballstem/english -# github.com/blevesearch/zap/v11 v11.0.14 -github.com/blevesearch/zap/v11 -# github.com/blevesearch/zap/v12 v12.0.14 -github.com/blevesearch/zap/v12 -# github.com/blevesearch/zap/v13 v13.0.6 -github.com/blevesearch/zap/v13 -# github.com/blevesearch/zap/v14 v14.0.5 -github.com/blevesearch/zap/v14 -# github.com/blevesearch/zap/v15 v15.0.3 -github.com/blevesearch/zap/v15 +# github.com/blevesearch/upsidedown_store_api v1.0.1 +github.com/blevesearch/upsidedown_store_api +# github.com/blevesearch/vellum v1.0.5 +github.com/blevesearch/vellum +github.com/blevesearch/vellum/levenshtein +github.com/blevesearch/vellum/regexp +github.com/blevesearch/vellum/utf8 +# github.com/blevesearch/zapx/v11 v11.2.2 +github.com/blevesearch/zapx/v11 +# github.com/blevesearch/zapx/v12 v12.2.2 +github.com/blevesearch/zapx/v12 +# github.com/blevesearch/zapx/v13 v13.2.2 +github.com/blevesearch/zapx/v13 +# github.com/blevesearch/zapx/v14 v14.2.2 +github.com/blevesearch/zapx/v14 +# github.com/blevesearch/zapx/v15 v15.2.2 +github.com/blevesearch/zapx/v15 # github.com/cespare/xxhash/v2 v2.1.1 ## explicit github.com/cespare/xxhash/v2 -# github.com/couchbase/vellum v1.0.2 -github.com/couchbase/vellum -github.com/couchbase/vellum/levenshtein -github.com/couchbase/vellum/regexp -github.com/couchbase/vellum/utf8 # github.com/davecgh/go-spew v1.1.1 github.com/davecgh/go-spew/spew # github.com/dgrijalva/jwt-go v3.2.0+incompatible @@ -678,9 +681,6 @@ github.com/vmihailenco/tagparser/v2/internal/parser github.com/wiggin77/merror # github.com/wiggin77/srslog v1.0.1 github.com/wiggin77/srslog -# github.com/willf/bitset v1.1.11 -## explicit -github.com/willf/bitset # github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 github.com/xi2/xz # github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c