Document extractor service (#15665)

* Document extractor service * Fixing vendor modules * Addressing PR Review comments * Some small simplifications * Fixing a linter complain * simplifying a bit the code using package variables Co-authored-by: Mattermod <mattermod@users.noreply.github.com>
2020-10-27 15:58:38 +01:00
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -4488,3 +4488,328 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.

+---
+
+## Archiver
+
+This product contains 'archiver' by Matthew Holt
+
+A library to handle direferen archive files (zip, rar, tar.gz...)
+
+* HOMEPAGE:
+  * https://github.com/mholt/archiver
+
+* LICENSE:
+
+MIT License
+
+Copyright (c) 2016 Matthew Holt
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+---
+
+## PDF Reader library
+
+This product contains 'pdf' by the Go team and modified by Thuc Le
+
+A library to provide pdf reading support
+
+* HOMEPAGE:
+  * https://github.com/ledongthuc/pdf
+
+* LICENSE:
+
+Copyright (c) 2009 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+---
+
+## GoOse
+
+This product contains 'GoOse' by Antonio Linari
+
+A library to provide html text extraction support
+
+* HOMEPAGE:
+  * https://github.com/advancedlogic/GoOse
+
+* LICENSE:
+
+Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+---
+
+## Docconv
+
+This product contains 'docconv' by Sajari Pty Ltd
+
+A library to provide text extraction support for different documents
+
+* HOMEPAGE:
+  * https://github.com/sajari/docconv
+
+* LICENSE:
+
+The MIT License (MIT)
+
+Copyright (c) 2014 Sajari Pty Ltd
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
--- a/go.mod
+++ b/go.mod
@@ -3,11 +3,13 @@ module github.com/mattermost/mattermost-server/v5
 go 1.14

 require (
+	code.sajari.com/docconv v1.1.1-0.20200701232649-d9ea05fbd50a
 	github.com/HdrHistogram/hdrhistogram-go v0.9.0 // indirect
 	github.com/Masterminds/semver/v3 v3.1.0
 	github.com/Masterminds/squirrel v1.4.0
 	github.com/NYTimes/gziphandler v1.1.1
 	github.com/RoaringBitmap/roaring v0.5.1 // indirect
+	github.com/advancedlogic/GoOse v0.0.0-20200830213114-1225d531e0ad // indirect
 	github.com/armon/go-metrics v0.3.4 // indirect
 	github.com/avct/uasurfer v0.0.0-20191028135549-26b5daa857f1
 	github.com/aws/aws-sdk-go v1.35.5
@@ -59,6 +61,7 @@ require (
 	github.com/jmoiron/sqlx v1.2.0
 	github.com/jonboulle/clockwork v0.2.2
 	github.com/klauspost/compress v1.11.1 // indirect
+	github.com/ledongthuc/pdf v0.0.0-20200323191019-23c5852adbd2
 	github.com/lib/pq v1.8.0
 	github.com/magiconair/properties v1.8.4 // indirect
 	github.com/mailru/easyjson v0.7.6 // indirect
@@ -70,15 +73,14 @@ require (
 	github.com/mattermost/rsc v0.0.0-20160330161541-bbaefb05eaa0
 	github.com/mattermost/viper v1.0.4
 	github.com/mattn/go-colorable v0.1.8 // indirect
-	github.com/mattn/go-runewidth v0.0.9 // indirect
 	github.com/mattn/go-sqlite3 v2.0.3+incompatible
+	github.com/mholt/archiver/v3 v3.4.0
 	github.com/miekg/dns v1.1.31 // indirect
 	github.com/minio/minio-go/v7 v7.0.5
 	github.com/mitchellh/go-testing-interface v1.14.1 // indirect
 	github.com/mitchellh/mapstructure v1.3.3 // indirect
 	github.com/muesli/smartcrop v0.3.0 // indirect
 	github.com/oklog/run v1.1.0 // indirect
-	github.com/olekukonko/tablewriter v0.0.4 // indirect
 	github.com/olivere/elastic v6.2.35+incompatible // indirect
 	github.com/oov/psd v0.0.0-20201002182931-74231384897f
 	github.com/opentracing/opentracing-go v1.2.0
@@ -103,7 +105,6 @@ require (
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
 	github.com/stretchr/objx v0.3.0 // indirect
 	github.com/stretchr/testify v1.6.1
 	github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect
--- a/go.sum
+++ b/go.sum
@@ -3,6 +3,9 @@ cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo=
 cloud.google.com/go v0.37.1/go.mod h1:SAbnLi6YTSPKSI0dTUEOVLCkyPfKXK8n4ibqiMoj4ok=
+code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc=
+code.sajari.com/docconv v1.1.1-0.20200701232649-d9ea05fbd50a h1:e0dRHyAfGcext3/lnwfsl81PKJRNgLyA843n72mPCes=
+code.sajari.com/docconv v1.1.1-0.20200701232649-d9ea05fbd50a/go.mod h1:DooS873W9YwUjTwEYGpg55aDlvnx1VcEdr7IJ9rEW8g=
 contrib.go.opencensus.io/exporter/ocagent v0.4.9/go.mod h1:ueLzZcP7LPhPulEBukGn4aLh7Mx9YJwpVJ9nL2FYltw=
 dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU=
 dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU=
@@ -20,6 +23,8 @@ github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mo
 github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/HdrHistogram/hdrhistogram-go v0.9.0 h1:dpujRju0R4M/QZzcnR1LH1qm+TVG3UzkWdp5tH1WMcg=
 github.com/HdrHistogram/hdrhistogram-go v0.9.0/go.mod h1:nxrse8/Tzg2tg3DZcZjm6qEclQKK70g0KxO61gFFZD4=
+github.com/JalfResi/justext v0.0.0-20170829062021-c0282dea7198 h1:8P+AjBhGByCuCX2zTkAf6UY+dj0JczX+t6cSdCSyvfw=
+github.com/JalfResi/justext v0.0.0-20170829062021-c0282dea7198/go.mod h1:0SURuH1rsE8aVWvutuMZghRNrNrYEUzibzJfhEYR8L0=
 github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY=
 github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM=
 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
@@ -33,6 +38,10 @@ github.com/Masterminds/squirrel v1.4.0/go.mod h1:yaPeOnPG5ZRwL9oKdTsO/prlkPbXWZl
 github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/PaulARoy/azurestoragecache v0.0.0-20170906084534-3c249a3ba788/go.mod h1:lY1dZd8HBzJ10eqKERHn3CU59tfhzcAVb2c0ZhIWSOk=
+github.com/PuerkitoBio/goquery v1.4.1 h1:smcIRGdYm/w7JSbcdeLHEMzxmsBQvl8lhf0dSw2nzMI=
+github.com/PuerkitoBio/goquery v1.4.1/go.mod h1:T9ezsOHcCrDCgA8aF1Cqr3sSYbO/xgdy8/R/XiIMAhA=
+github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
+github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
 github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo=
 github.com/RoaringBitmap/roaring v0.5.1 h1:ugdwntNygzk1FZnmtxUr+jM9AYrpU3I3zpt49npDWVo=
 github.com/RoaringBitmap/roaring v0.5.1/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo=
@@ -40,6 +49,9 @@ github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqR
 github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
 github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
+github.com/advancedlogic/GoOse v0.0.0-20191112112754-e742535969c1/go.mod h1:f3HCSN1fBWjcpGtXyM119MJgeQl838v6so/PQOqvE1w=
+github.com/advancedlogic/GoOse v0.0.0-20200830213114-1225d531e0ad h1:gyzOmx++wVkSj5kLzYtvNN2ooeJGTFTtV37t5Do4sdM=
+github.com/advancedlogic/GoOse v0.0.0-20200830213114-1225d531e0ad/go.mod h1:f3HCSN1fBWjcpGtXyM119MJgeQl838v6so/PQOqvE1w=
 github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
 github.com/agnivade/gziphandler v1.1.2-0.20200815170021-7481835cb745 h1:K5MCP8PQt50Tqkcdtgg7jhYwo6VBJWMKUmgqbXdu7WE=
 github.com/agnivade/gziphandler v1.1.2-0.20200815170021-7481835cb745/go.mod h1:eRIEzSVYGdp3PyjpBJPBPMo4LSeFUlpPjByDCzMRrtw=
@@ -49,9 +61,20 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
+github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4=
+github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
+github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
+github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/andybalholm/cascadia v1.2.0 h1:vuRCkM5Ozh/BfmsaTm26kbjm0mIOM3yS5Ek/F5h18aE=
+github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
+github.com/araddon/dateparse v0.0.0-20180729174819-cfd92a431d0e h1:s05JG2GwtJMHaPcXDpo4V35TFgyYZzNsmBlSkHPEbeg=
+github.com/araddon/dateparse v0.0.0-20180729174819-cfd92a431d0e/go.mod h1:SLqhdZcd+dF3TEVL2RMoob5bBP5R1P1qkox+HtCBgGI=
+github.com/araddon/dateparse v0.0.0-20200409225146-d820a6159ab1 h1:TEBmxO80TM04L8IuMWk77SGL1HomBmKTdzdJLLWznxI=
+github.com/araddon/dateparse v0.0.0-20200409225146-d820a6159ab1/go.mod h1:SLqhdZcd+dF3TEVL2RMoob5bBP5R1P1qkox+HtCBgGI=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I=
@@ -143,6 +166,7 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d h1:SwD98825d6bdB+pEuTxWOXiSjBrHdOl/UVp75eI7JT8=
 github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8=
 github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso=
@@ -164,6 +188,9 @@ github.com/disintegration/imaging v1.6.0/go.mod h1:xuIt+sRxDFrHS0drzXUlCJthkJ8k7
 github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
 github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
 github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
+github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q=
+github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo=
+github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dyatlov/go-opengraph v0.0.0-20180429202543-816b6608b3c8 h1:6muCmMJat6z7qptVrIf/+OWPxsjAfvhw5/6t+FwEkgg=
@@ -192,6 +219,8 @@ github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
+github.com/fatih/set v0.2.1 h1:nn2CaJyknWE/6txyUDGwysr3G5QC6xWB/PtVjPBbeaA=
+github.com/fatih/set v0.2.1/go.mod h1:+RKtMCH+favT2+3YecHGxcc0b4KyVWA1QWWJUs4E0CI=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ=
 github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
@@ -203,6 +232,9 @@ github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJn
 github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
 github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
 github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=
+github.com/frankban/quicktest v1.4.0/go.mod h1:36zfPVQyHxymz4cH7wlDmVwDrJuljRB60qkgn7rorfQ=
+github.com/frankban/quicktest v1.10.0 h1:Gfh+GAJZOAoKZsIZeZbdn2JF10kN1XHNvjsvQK8gVkE=
+github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
@@ -212,6 +244,8 @@ github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv
 github.com/getsentry/sentry-go v0.7.0 h1:MR2yfR4vFfv/2+iBuSnkdQwVg7N9cJzihZ6KJu7srwQ=
 github.com/getsentry/sentry-go v0.7.0/go.mod h1:pLFpD2Y5RHIKF9Bw3KH6/68DeN2K/XBJd8awjdPnUwg=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/gigawattio/window v0.0.0-20180317192513-0f5467e35573 h1:u8AQ9bPa9oC+8/A/jlWouakhIvkFfuxgIIRjiy8av7I=
+github.com/gigawattio/window v0.0.0-20180317192513-0f5467e35573/go.mod h1:eBvb3i++NHDH4Ugo9qCvMw8t0mTSctaEa5blJbWcNxs=
 github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s=
 github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM=
 github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
@@ -235,6 +269,10 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
 github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8=
+github.com/go-resty/resty/v2 v2.0.0 h1:9Nq/U+V4xsoDnDa/iTrABDWUCuk3Ne92XFHPe6dKWUc=
+github.com/go-resty/resty/v2 v2.0.0/go.mod h1:dZGr0i9PLlaaTD4H/hoZIDjQ+r6xq8mgbRzHZf7f2J8=
+github.com/go-resty/resty/v2 v2.3.0 h1:JOOeAvjSlapTT92p8xiS19Zxev1neGikoHsXJeOq8So=
+github.com/go-resty/resty/v2 v2.3.0/go.mod h1:UpN9CgLZNsv4e9XG50UU8xdI0F43UQ4HmxLBDwaroHU=
 github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
@@ -291,6 +329,8 @@ github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -408,6 +448,7 @@ github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/
 github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI=
 github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw=
 github.com/jamiealquiza/envy v1.1.0/go.mod h1:MP36BriGCLwEHhi1OU8E9569JNZrjWfCvzG7RsPnHus=
+github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk=
 github.com/jaytaylor/html2text v0.0.0-20200412013138-3577fbdbcff7 h1:g0fAGBisHaEQ0TRq1iBvemFRf+8AEWEmBESSiWB3Vsc=
 github.com/jaytaylor/html2text v0.0.0-20200412013138-3577fbdbcff7/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk=
 github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
@@ -453,17 +494,22 @@ github.com/kataras/neffos v0.0.10/go.mod h1:ZYmJC07hQPW67eKuzlfY7SO3bC0mw83A3j6i
 github.com/kataras/pio v0.0.0-20190103105442-ea782b38602d/go.mod h1:NV88laa9UiiDuX9AhMbDPkGYSPugBOV6yTZB1l2K9Z0=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.10.11 h1:K9z59aO18Aywg2b/WSgBaUX99mHy2BES18Cr5lBKZHk=
 github.com/klauspost/compress v1.10.11/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.11.1 h1:bPb7nMRdOZYDrpPMTA3EInUQrdgoBinqUuSwlGdKDdE=
 github.com/klauspost/compress v1.11.1/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/cpuid v1.2.3 h1:CCtW0xUnWGVINKvE/WWOYKdsPV6mawAtvQuSl8guwQs=
 github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=
 github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
+github.com/klauspost/pgzip v1.2.4 h1:TQ7CNpYKovDOmqzRHKxJh0BeaBI7UdQZYc6p7pMQh1A=
+github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
 github.com/kljensen/snowball v0.6.0/go.mod h1:27N7E8fVU5H68RlUmnWwZCfxgt4POBJfENGMvNRhldw=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -473,16 +519,24 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
+github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g=
 github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
 github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw=
 github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o=
 github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk=
 github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw=
+github.com/ledongthuc/pdf v0.0.0-20200323191019-23c5852adbd2 h1:H9HhyvygtvWnn1R8ymra4vdIUOvDDlaPlX6mjoJ9UTY=
+github.com/ledongthuc/pdf v0.0.0-20200323191019-23c5852adbd2/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
+github.com/levigross/exp-html v0.0.0-20120902181939-8df60c69a8f5 h1:W7p+m/AECTL3s/YR5RpQ4hz5SjNeKzZBl1q36ws12s0=
+github.com/levigross/exp-html v0.0.0-20120902181939-8df60c69a8f5/go.mod h1:QMe2wuKJ0o7zIVE8AqiT8rd8epmm6WDIZ2wyuBqYPzM=
 github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A=
 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
@@ -531,6 +585,7 @@ github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOA
 github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
@@ -542,6 +597,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0j
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg=
 github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ=
+github.com/mholt/archiver/v3 v3.4.0 h1:XnyitVjWu0seab2U0gczXEmVcom1UaE+tA1FCkN3814=
+github.com/mholt/archiver/v3 v3.4.0/go.mod h1:00RcBMhNszoMqnda/5VHHL9+cBoTgj+AkLnmwr3HGrA=
 github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
 github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
@@ -555,6 +612,7 @@ github.com/minio/minio-go/v7 v7.0.5/go.mod h1:TA0CQCjJZHM5SJj9IjqR0NmpmQJ6bCbXif
 github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
 github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
@@ -601,6 +659,8 @@ github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
 github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM=
+github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ=
+github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0=
 github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs=
@@ -610,6 +670,7 @@ github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA=
 github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
+github.com/olekukonko/tablewriter v0.0.0-20180506121414-d4647c9c7a84/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
 github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8=
 github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA=
 github.com/olivere/elastic v6.2.35+incompatible h1:MMklYDy2ySi01s123CB2WLBuDMzFX4qhFcA5tKWJPgM=
@@ -617,6 +678,7 @@ github.com/olivere/elastic v6.2.35+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGe
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.10.3 h1:OoxbjfXVZyod1fmWYhI7SEyaD8B00ynP3T+D5GiyHOY=
 github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
@@ -624,6 +686,7 @@ github.com/onsi/ginkgo v1.13.0 h1:M76yO2HkZASFjXL0HSoZJ1AYEmQxNJmY41Jx1zNUq1Y=
 github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0=
 github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1 h1:K0jcRCwNQM3vFGh1ppMtDh/+7ApJrjldlX8fA0jDTLQ=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
@@ -644,6 +707,12 @@ github.com/openzipkin/zipkin-go v0.1.3/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTm
 github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
 github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
 github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
+github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95 h1:+OLn68pqasWca0z5ryit9KGfp3sUsW4Lqg32iRMJyzs=
+github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
+github.com/otiai10/gosseract/v2 v2.2.4 h1:h/PV+oJqke8q2Ccw9bjpMBWfd7N2vtGDCUcihZj3nRo=
+github.com/otiai10/gosseract/v2 v2.2.4/go.mod h1:ahOp/kHojnOMGv1RaUnR0jwY5JVa6BYKhYAS8nbMLSo=
+github.com/otiai10/mint v1.3.0 h1:Ady6MKVezQwHBkGzLFbrsywyp09Ah7rkmfjV3Bcr5uc=
+github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
 github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
@@ -661,8 +730,12 @@ github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9
 github.com/peterbourgon/diskv v0.0.0-20171120014656-2973218375c3/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ=
 github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
+github.com/pierrec/cmdflag v0.0.2/go.mod h1:a3zKGZ3cdQUfxjd0RGMLZr8xI3nvpJOB+m6o/1X5BmU=
 github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
+github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
+github.com/pierrec/lz4/v3 v3.3.2 h1:QTUOCbMNDbK4PYtkuHyOBd28C0UhPBw3T4OH4WpFDik=
+github.com/pierrec/lz4/v3 v3.3.2/go.mod h1:280XNCGS8jAcG++AHdd6SeWnzyJ1w9oow2vbORyey8Q=
 github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
 github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -747,6 +820,7 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb
 github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
 github.com/satori/go.uuid v0.0.0-20180103174451-36e9d2ebbde5/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
+github.com/schollz/progressbar/v2 v2.13.2/go.mod h1:6YZjqdthH6SCZKv2rqGryrxPtfmRB/DWZxSMfCXPyD8=
 github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
@@ -776,6 +850,8 @@ github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4=
 github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw=
+github.com/simplereach/timeutils v1.2.0 h1:btgOAlu9RW6de2r2qQiONhjgxdAG7BL6je0G6J/yPnA=
+github.com/simplereach/timeutils v1.2.0/go.mod h1:VVbQDfN/FHRZa1LSqcwo4kNZ62OOyqLLGQKYB3pB0Q8=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
@@ -863,6 +939,9 @@ github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVM
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
+github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
+github.com/ulikunitz/xz v0.5.7 h1:YvTNdFzX6+W5m9msiYg/zpkSURPPtOlzbqYjrFn7Yt4=
+github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4=
@@ -892,6 +971,8 @@ github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
+github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=
+github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c h1:3lbZUMbMiGUW/LMkfsEABsc5zNT9+b1CvsJx47JzJ8g=
@@ -976,6 +1057,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee h1:WG0RUwxtNT4qqaXX3DPA8zH
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -998,12 +1080,15 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
@@ -1203,6 +1288,8 @@ gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/mail.v2 v2.3.1 h1:WYFn/oANrAGP2C0dcV6/pbkPzv8yGzqTjPmTeO7qoXk=
 gopkg.in/mail.v2 v2.3.1/go.mod h1:htwXN1Qh09vZJ1NVKxQqHPBaCBbzKhp5GzuJEA4VJWw=
 gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/olivere/elastic.v6 v6.2.35 h1:/5dJ0UzM231DGl9eDYOdDgv8yCFzAHQVPMo69rnylks=
--- a/services/docextractor/archive.go
+++ b/services/docextractor/archive.go
@@ -0,0 +1,60 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/mholt/archiver/v3"
+)
+
+type archiveExtractor struct {
+	SubExtractor Extractor
+}
+
+func (ae *archiveExtractor) Match(filename string) bool {
+	_, err := archiver.ByExtension(filename)
+	return err == nil
+}
+
+func (ae *archiveExtractor) Extract(name string, r io.Reader) (string, error) {
+	dir, err := ioutil.TempDir(os.TempDir(), "archiver")
+	if err != nil {
+		return "", fmt.Errorf("error creating temporary file: %v", err)
+	}
+	defer os.RemoveAll(dir)
+
+	f, err := os.Create(filepath.Join(dir, name))
+	if err != nil {
+		return "", fmt.Errorf("error copying data into temporary file: %v", err)
+	}
+	_, err = io.Copy(f, r)
+	f.Close()
+	if err != nil {
+		return "", fmt.Errorf("error copying data into temporary file: %v", err)
+	}
+
+	var text strings.Builder
+	err = archiver.Walk(f.Name(), func(file archiver.File) error {
+		text.WriteString(file.Name() + " ")
+		if ae.SubExtractor != nil {
+			filename := filepath.Base(file.Name())
+			subtext, extractErr := ae.SubExtractor.Extract(filename, file)
+			if extractErr == nil {
+				text.WriteString(subtext + " ")
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return "", err
+	}
+
+	return text.String(), nil
+}
--- a/services/docextractor/combine.go
+++ b/services/docextractor/combine.go
@@ -0,0 +1,41 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"io"
+
+	"github.com/mattermost/mattermost-server/v5/mlog"
+)
+
+type combineExtractor struct {
+	SubExtractors []Extractor
+}
+
+func (ce *combineExtractor) Add(extractor Extractor) {
+	ce.SubExtractors = append(ce.SubExtractors, extractor)
+}
+
+func (ce *combineExtractor) Match(filename string) bool {
+	for _, extractor := range ce.SubExtractors {
+		if extractor.Match(filename) {
+			return true
+		}
+	}
+	return false
+}
+
+func (ce *combineExtractor) Extract(filename string, r io.Reader) (string, error) {
+	for _, extractor := range ce.SubExtractors {
+		if extractor.Match(filename) {
+			text, err := extractor.Extract(filename, r)
+			if err != nil {
+				mlog.Warn("unable to extract file content", mlog.Err(err))
+				continue
+			}
+			return text, nil
+		}
+	}
+	return "", nil
+}
--- a/services/docextractor/docextractor.go
+++ b/services/docextractor/docextractor.go
@@ -0,0 +1,46 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"io"
+)
+
+// ExtractSettings defines the features enabled/disable during the document text extraction.
+type ExtractSettings struct {
+	ArchiveRecursion bool
+	MMPreviewURL     string
+	MMPreviewSecret  string
+}
+
+// Extract extract the text from a document using the system default extractors
+func Extract(filename string, r io.Reader, settings ExtractSettings) (string, error) {
+	return ExtractWithExtraExtractors(filename, r, settings, []Extractor{})
+}
+
+// ExtractWithExtraExtractors extract the text from a document using the provided extractors beside the system default extractors.
+func ExtractWithExtraExtractors(filename string, r io.Reader, settings ExtractSettings, extraExtractors []Extractor) (string, error) {
+	enabledExtractors := &combineExtractor{}
+	for _, extraExtractor := range extraExtractors {
+		enabledExtractors.Add(extraExtractor)
+	}
+	enabledExtractors.Add(&pdfExtractor{})
+	enabledExtractors.Add(&documentExtractor{})
+
+	if settings.ArchiveRecursion {
+		enabledExtractors.Add(&archiveExtractor{SubExtractor: enabledExtractors})
+	} else {
+		enabledExtractors.Add(&archiveExtractor{})
+	}
+
+	if settings.MMPreviewURL != "" {
+		enabledExtractors.Add(newMMPreviewExtractor(settings.MMPreviewURL, settings.MMPreviewSecret, pdfExtractor{}))
+	}
+	enabledExtractors.Add(&plainExtractor{})
+
+	if enabledExtractors.Match(filename) {
+		return enabledExtractors.Extract(filename, r)
+	}
+	return "", nil
+}
--- a/services/docextractor/docextractor_test.go
+++ b/services/docextractor/docextractor_test.go
@@ -0,0 +1,186 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost-server/v5/utils/testutils"
+)
+
+func TestExtract(t *testing.T) {
+	testCases := []struct {
+		Name         string
+		TestFileName string
+		Settings     ExtractSettings
+		Contains     []string
+		NotContains  []string
+		ExpectError  bool
+	}{
+		{
+			"Plain text file",
+			"test-markdown-basics.md",
+			ExtractSettings{},
+			[]string{"followed", "separated"},
+			[]string{},
+			false,
+		},
+		{
+			"Zip file without recursion",
+			"Fake_Team_Import.zip",
+			ExtractSettings{},
+			[]string{"users", "channels", "general"},
+			[]string{"purpose", "announcements"},
+			false,
+		},
+		{
+			"Zip file with recursion",
+			"Fake_Team_Import.zip",
+			ExtractSettings{ArchiveRecursion: true},
+			[]string{"users", "channels", "general", "purpose", "announcements"},
+			[]string{},
+			false,
+		},
+		{
+			"Rar file without recursion",
+			"Fake_Team_Import.rar",
+			ExtractSettings{},
+			[]string{"users", "channels", "general"},
+			[]string{"purpose", "announcements"},
+			false,
+		},
+		{
+			"Rar file with recursion",
+			"Fake_Team_Import.rar",
+			ExtractSettings{ArchiveRecursion: true},
+			[]string{"users", "channels", "general", "purpose", "announcements"},
+			[]string{},
+			false,
+		},
+		{
+			"Tar.gz file without recursion",
+			"Fake_Team_Import.tar.gz",
+			ExtractSettings{},
+			[]string{"users", "channels", "general"},
+			[]string{"purpose", "announcements"},
+			false,
+		},
+		{
+			"Tar.gz file with recursion",
+			"Fake_Team_Import.tar.gz",
+			ExtractSettings{ArchiveRecursion: true},
+			[]string{"users", "channels", "general", "purpose", "announcements"},
+			[]string{},
+			false,
+		},
+		{
+			"Pdf file",
+			"sample-doc.pdf",
+			ExtractSettings{},
+			[]string{"simple", "document", "contains"},
+			[]string{},
+			false,
+		},
+		{
+			"Docx file",
+			"sample-doc.docx",
+			ExtractSettings{},
+			[]string{"simple", "document", "contains"},
+			[]string{},
+			false,
+		},
+		{
+			"Pptx file",
+			"sample-doc.pptx",
+			ExtractSettings{},
+			[]string{"simple", "document", "contains"},
+			[]string{},
+			false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.Name, func(t *testing.T) {
+			data, err := testutils.ReadTestFile(tc.TestFileName)
+			require.Nil(t, err)
+			text, err := Extract(tc.TestFileName, bytes.NewReader(data), tc.Settings)
+			if tc.ExpectError {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+				for _, expectedString := range tc.Contains {
+					assert.Contains(t, text, expectedString)
+				}
+				for _, notExpectedString := range tc.NotContains {
+					assert.NotContains(t, text, notExpectedString)
+				}
+			}
+		})
+	}
+
+	t.Run("Unsupported binary file", func(t *testing.T) {
+		data, err := testutils.ReadTestFile("testjpg.jpg")
+		require.Nil(t, err)
+		text, err := Extract("testjpg.jpg", bytes.NewReader(data), ExtractSettings{})
+		require.NoError(t, err)
+		require.Equal(t, "", text)
+	})
+
+	t.Run("Wrong extension", func(t *testing.T) {
+		data, err := testutils.ReadTestFile("sample-doc.pdf")
+		require.Nil(t, err)
+		text, err := Extract("sample-doc.docx", bytes.NewReader(data), ExtractSettings{})
+		require.NoError(t, err)
+		require.Equal(t, "", text)
+	})
+}
+
+type customTestPdfExtractor struct{}
+
+func (te *customTestPdfExtractor) Match(filename string) bool {
+	return strings.HasSuffix(filename, ".pdf")
+}
+
+func (te *customTestPdfExtractor) Extract(filename string, r io.Reader) (string, error) {
+	return "this is a text generated content", nil
+}
+
+type failingExtractor struct{}
+
+func (te *failingExtractor) Match(filename string) bool {
+	return true
+}
+
+func (te *failingExtractor) Extract(filename string, r io.Reader) (string, error) {
+	return "", errors.New("this always fail")
+}
+
+func TestExtractWithExtraExtractors(t *testing.T) {
+	t.Run("overrite existing extractor", func(t *testing.T) {
+		data, err := testutils.ReadTestFile("sample-doc.pdf")
+		require.Nil(t, err)
+
+		text, err := ExtractWithExtraExtractors("sample-doc.pdf", bytes.NewReader(data), ExtractSettings{}, []Extractor{&customTestPdfExtractor{}})
+		require.NoError(t, err)
+		require.Equal(t, text, "this is a text generated content")
+	})
+
+	t.Run("failing extractor", func(t *testing.T) {
+		data, err := testutils.ReadTestFile("sample-doc.pdf")
+		require.Nil(t, err)
+
+		text, err := ExtractWithExtraExtractors("sample-doc.pdf", bytes.NewReader(data), ExtractSettings{}, []Extractor{&failingExtractor{}})
+		require.NoError(t, err)
+		assert.Contains(t, text, "simple")
+		assert.Contains(t, text, "document")
+		assert.Contains(t, text, "contains")
+	})
+}
--- a/services/docextractor/documents.go
+++ b/services/docextractor/documents.go
@@ -0,0 +1,61 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"strings"
+
+	"code.sajari.com/docconv"
+)
+
+type documentExtractor struct{}
+
+var doconvConverterByExtensions = map[string]func(io.Reader) (string, map[string]string, error){
+	"doc":   docconv.ConvertDoc,
+	"docx":  docconv.ConvertDocx,
+	"pptx":  docconv.ConvertPptx,
+	"odt":   docconv.ConvertODT,
+	"html":  func(r io.Reader) (string, map[string]string, error) { return docconv.ConvertHTML(r, true) },
+	"pages": docconv.ConvertPages,
+	"rtf":   docconv.ConvertRTF,
+}
+
+func (de *documentExtractor) Match(filename string) bool {
+	extension := strings.TrimPrefix(path.Ext(filename), ".")
+	_, ok := doconvConverterByExtensions[extension]
+	return ok
+}
+
+func (de *documentExtractor) Extract(filename string, r io.Reader) (string, error) {
+	extension := strings.TrimPrefix(path.Ext(filename), ".")
+	converter, ok := doconvConverterByExtensions[extension]
+	if !ok {
+		return "", errors.New("Unknown converter")
+	}
+
+	f, err := ioutil.TempFile(os.TempDir(), "docconv")
+	if err != nil {
+		return "", fmt.Errorf("error creating temporary file: %v", err)
+	}
+	defer f.Close()
+	defer os.Remove(f.Name())
+
+	_, err = io.Copy(f, r)
+	if err != nil {
+		return "", fmt.Errorf("error copying data into temporary file: %v", err)
+	}
+
+	text, _, err := converter(f)
+	if err != nil {
+		return "", err
+	}
+
+	return text, nil
+}
--- a/services/docextractor/interface.go
+++ b/services/docextractor/interface.go
@@ -0,0 +1,14 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"io"
+)
+
+// Extractors define the interface needed to extract file content
+type Extractor interface {
+	Match(filename string) bool
+	Extract(filename string, file io.Reader) (string, error)
+}
--- a/services/docextractor/mmpreview.go
+++ b/services/docextractor/mmpreview.go
@@ -0,0 +1,81 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+// MMPreview is a micro-service to convert from any libreoffice supported
+// format into a PDF file, and then we use the regular pdf extractor to convert
+// it into plain text.
+
+import (
+	"bytes"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"path"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+type mmPreviewExtractor struct {
+	url          string
+	secret       string
+	pdfExtractor pdfExtractor
+}
+
+var mmpreviewSupportedExtensions = map[string]bool{
+	"ppt":  true,
+	"odp":  true,
+	"xls":  true,
+	"xlsx": true,
+	"ods":  true,
+}
+
+func newMMPreviewExtractor(url string, secret string, pdfExtractor pdfExtractor) *mmPreviewExtractor {
+	return &mmPreviewExtractor{url: url, secret: secret, pdfExtractor: pdfExtractor}
+}
+
+func (mpe *mmPreviewExtractor) Match(filename string) bool {
+	extension := strings.TrimPrefix(path.Ext(filename), ".")
+	return mmpreviewSupportedExtensions[extension]
+}
+
+func (mpe *mmPreviewExtractor) Extract(filename string, file io.Reader) (string, error) {
+	b, w, err := createMultipartFormData("file", filename, file)
+	if err != nil {
+		return "", errors.Wrap(err, "Unable to generate file preview using mmpreview.")
+	}
+	req, err := http.NewRequest("POST", mpe.url+"/toPDF", &b)
+	if err != nil {
+		return "", errors.Wrap(err, "Unable to generate file preview using mmpreview.")
+	}
+	req.Header.Set("Content-Type", w.FormDataContentType())
+	if mpe.secret != "" {
+		req.Header.Add("Authentication", mpe.secret)
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", errors.Wrap(err, "Unable to generate file preview using mmpreview.")
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		return "", errors.New("Unable to generate file preview using mmpreview (The server has replied with an error)")
+	}
+	return mpe.pdfExtractor.Extract(filename, resp.Body)
+}
+
+func createMultipartFormData(fieldName, fileName string, fileData io.Reader) (bytes.Buffer, *multipart.Writer, error) {
+	var b bytes.Buffer
+	var err error
+	w := multipart.NewWriter(&b)
+	var fw io.Writer
+	if fw, err = w.CreateFormFile(fieldName, fileName); err != nil {
+		return b, nil, err
+	}
+	if _, err = io.Copy(fw, fileData); err != nil {
+		return b, nil, err
+	}
+	w.Close()
+	return b, w, nil
+}
--- a/services/docextractor/pdf.go
+++ b/services/docextractor/pdf.go
@@ -0,0 +1,52 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/ledongthuc/pdf"
+)
+
+type pdfExtractor struct{}
+
+func (pe *pdfExtractor) Match(filename string) bool {
+	supportedExtensions := map[string]bool{
+		"pdf": true,
+	}
+	extension := strings.TrimPrefix(path.Ext(filename), ".")
+	return supportedExtensions[extension]
+}
+
+func (pe *pdfExtractor) Extract(filename string, r io.Reader) (string, error) {
+	f, err := ioutil.TempFile(os.TempDir(), "pdflib")
+	if err != nil {
+		return "", fmt.Errorf("error creating temporary file: %v", err)
+	}
+	defer f.Close()
+	defer os.Remove(f.Name())
+	size, err := io.Copy(f, r)
+	if err != nil {
+		return "", fmt.Errorf("error copying data into temporary file: %v", err)
+	}
+
+	reader, err := pdf.NewReader(f, size)
+	if err != nil {
+		return "", err
+	}
+
+	var buf bytes.Buffer
+	b, err := reader.GetPlainText()
+	if err != nil {
+		return "", err
+	}
+	buf.ReadFrom(b)
+	return buf.String(), nil
+}
--- a/services/docextractor/plain.go
+++ b/services/docextractor/plain.go
@@ -0,0 +1,40 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"io"
+	"io/ioutil"
+	"unicode"
+	"unicode/utf8"
+)
+
+type plainExtractor struct{}
+
+func (pe *plainExtractor) Match(filename string) bool {
+	return true
+}
+
+func (pe *plainExtractor) Extract(filename string, r io.Reader) (string, error) {
+	// This detects any visible character plus any whitespace
+	validRanges := append(unicode.GraphicRanges, unicode.White_Space)
+
+	text, _ := ioutil.ReadAll(r)
+	count := 0
+	for {
+		c, size := utf8.DecodeRune(text[count:])
+		if !unicode.In(c, validRanges...) {
+			return "", nil
+		}
+		if size == 0 {
+			break
+		}
+		count += size
+		if count > 1024 {
+			break
+		}
+	}
+
+	return string(text), nil
+}
--- a/tests/Fake_Team_Import.rar
+++ b/tests/Fake_Team_Import.rar
--- a/tests/Fake_Team_Import.tar.gz
+++ b/tests/Fake_Team_Import.tar.gz
--- a/tests/sample-doc.docx
+++ b/tests/sample-doc.docx
--- a/tests/sample-doc.odp
+++ b/tests/sample-doc.odp
--- a/tests/sample-doc.odt
+++ b/tests/sample-doc.odt
--- a/tests/sample-doc.pdf
+++ b/tests/sample-doc.pdf
--- a/tests/sample-doc.pptx
+++ b/tests/sample-doc.pptx
--- a/tests/sample-doc.rtf
+++ b/tests/sample-doc.rtf
@@ -0,0 +1,17 @@
+{\rtf1\ansi\deff3\adeflang1025
+{\fonttbl{\f0\froman\fprq2\fcharset0 Times New Roman;}{\f1\froman\fprq2\fcharset2 Symbol;}{\f2\fswiss\fprq2\fcharset0 Arial;}{\f3\froman\fprq2\fcharset0 Liberation Serif{\*\falt Times New Roman};}{\f4\froman\fprq2\fcharset0 Liberation Sans{\*\falt Arial};}{\f5\fnil\fprq2\fcharset0 Noto Serif CJK SC;}{\f6\fnil\fprq2\fcharset0 Noto Sans CJK SC;}{\f7\fnil\fprq2\fcharset0 FreeSans;}}
+{\colortbl;\red0\green0\blue0;\red0\green0\blue255;\red0\green255\blue255;\red0\green255\blue0;\red255\green0\blue255;\red255\green0\blue0;\red255\green255\blue0;\red255\green255\blue255;\red0\green0\blue128;\red0\green128\blue128;\red0\green128\blue0;\red128\green0\blue128;\red128\green0\blue0;\red128\green128\blue0;\red128\green128\blue128;\red192\green192\blue192;}
+{\stylesheet{\s0\snext0\dbch\af5\langfe2052\dbch\af7\afs24\alang1081\ql\widctlpar\hyphpar0\sb0\sa0\ltrpar\cf0\loch\f3\fs24\lang1033\kerning1 Normal;}
+{\s15\sbasedon0\snext16\dbch\af6\langfe2052\dbch\af7\afs28\alang1081\ql\widctlpar\hyphpar0\sb240\sa120\keepn\ltrpar\cf0\loch\f4\fs28\lang1033\kerning1 Heading;}
+{\s16\sbasedon0\snext16\dbch\af5\langfe2052\dbch\af7\afs24\alang1081\sl276\slmult1\ql\widctlpar\hyphpar0\sb0\sa140\ltrpar\cf0\loch\f3\fs24\lang1033\kerning1 Text Body;}
+{\s17\sbasedon16\snext17\dbch\af5\langfe2052\dbch\af7\afs24\alang1081\sl276\slmult1\ql\widctlpar\hyphpar0\sb0\sa140\ltrpar\cf0\loch\f3\fs24\lang1033\kerning1 List;}
+{\s18\sbasedon0\snext18\dbch\af5\langfe2052\dbch\af7\afs24\alang1081\ai\ql\widctlpar\hyphpar0\sb120\sa120\noline\ltrpar\cf0\loch\f3\fs24\lang1033\i\kerning1 Caption;}
+{\s19\sbasedon0\snext19\dbch\af5\langfe2052\dbch\af7\afs24\alang1081\ql\widctlpar\hyphpar0\sb0\sa0\noline\ltrpar\cf0\loch\f3\fs24\lang1033\kerning1 Index;}
+}{\*\generator LibreOffice/7.0.1.2$Linux_X86_64 LibreOffice_project/00$Build-2}{\info{\creatim\yr2020\mo9\dy25\hr10\min37}{\revtim\yr2020\mo9\dy25\hr10\min38}{\printim\yr0\mo0\dy0\hr0\min0}}{\*\userprops}\deftab709
+\hyphauto1\viewscale180
+{\*\pgdsctbl
+{\pgdsc0\pgdscuse451\pgwsxn12240\pghsxn15840\marglsxn1134\margrsxn1134\margtsxn1134\margbsxn1134\pgdscnxt0 Default Page Style;}}
+\formshade{\*\pgdscno0}\paperh15840\paperw12240\margl1134\margr1134\margt1134\margb1134\sectd\sbknone\pgndec\sftnnar\saftnnrlc\sectunlocked1\pgwsxn12240\pghsxn15840\marglsxn1134\margrsxn1134\margtsxn1134\margbsxn1134\ftnbj\ftnstart1\ftnrstcont\ftnnar\aenddoc\aftnrstcont\aftnstart1\aftnnrlc\htmautsp
+{\*\ftnsep\chftnsep}\pgndec\pard\plain \s0\dbch\af5\langfe2052\dbch\af7\afs24\alang1081\ql\widctlpar\hyphpar0\sb0\sa0\ltrpar\cf0\loch\f3\fs24\lang1033\kerning1\ql\ltrpar{\loch
+This is a simple document that contains some text.}
+\par }
--- a/vendor/code.sajari.com/docconv/.gitignore
+++ b/vendor/code.sajari.com/docconv/.gitignore
@@ -0,0 +1,3 @@
+
+sajari-convert
+*tests/
--- a/vendor/code.sajari.com/docconv/.travis.yml
+++ b/vendor/code.sajari.com/docconv/.travis.yml
@@ -0,0 +1,10 @@
+sudo: false
+language: go
+go:
+- "1.13"
+- "1.14"
+- tip
+go_import_path: code.sajari.com/docconv
+notifications:
+  email:
+    - infra@sajari.com
--- a/vendor/code.sajari.com/docconv/LICENSE
+++ b/vendor/code.sajari.com/docconv/LICENSE
@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Sajari Pty Ltd
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
--- a/vendor/code.sajari.com/docconv/README.md
+++ b/vendor/code.sajari.com/docconv/README.md
@@ -0,0 +1,139 @@
+# docconv
+
+[![GoDoc](https://godoc.org/code.sajari.com/docconv?status.svg)](https://godoc.org/code.sajari.com/docconv)
+[![Build Status](https://travis-ci.org/sajari/docconv.svg?branch=master)](https://travis-ci.org/sajari/docconv)
+
+A Go wrapper library to convert PDF, DOC, DOCX, XML, HTML, RTF, ODT, Pages documents and images (see optional dependencies below) to plain text.
+
+> **Note for returning users:** the Go import path for this package been moved to `code.sajari.com/docconv`.
+
+## Installation
+
+If you haven't setup Go before, you first need to [install Go](https://golang.org/doc/install).
+
+To fetch and build the code:
+
+    $ go get code.sajari.com/docconv/...
+
+This will also build the command line tool `docd` into `$GOPATH/bin`. Make sure that `$GOPATH/bin` is in your `PATH` environment variable.
+
+## Dependencies
+
+tidy, wv, popplerutils, unrtf, https://github.com/JalfResi/justext
+
+Example install of dependencies (not all systems):
+
+    $ sudo apt-get install poppler-utils wv unrtf tidy
+    $ go get github.com/JalfResi/justext
+
+### Optional dependencies
+
+To add image support to the `docconv` library you first need to [install and build gosseract](https://github.com/otiai10/gosseract/tree/v2.2.4).
+
+Now you can add `-tags ocr` to any `go` command when building/fetching/testing `docconv` to include support for processing images:
+
+    $ go get -tags ocr code.sajari.com/docconv/...
+
+This may complain on macOS, which you can fix by installing [tesseract](https://tesseract-ocr.github.io) via brew:
+
+    $ brew install tesseract
+
+## docd tool
+
+The `docd` tool runs as either:
+
+1.  a service on port 8888 (by default)
+
+    Documents can be sent as a multipart POST request and the plain text (body) and meta information are then returned as a JSON object.
+
+2.  a service exposed from within a Docker container
+
+    This also runs as a service, but from within a Docker container. There are three build scripts:
+
+    - [./docd/debian.sh](./docd/debian.sh)
+    - [./docd/alpine.sh](./docd/alpine.sh)
+    - [./docd/appengine.sh](./docd/appengine.sh)
+
+    The `debian` version uses the Debian package repository which can vary with builds. The `alpine` version uses a very cut down Linux distribution to produce a container ~40MB. It also locks the dependency versions for consistency, but may miss out on future updates. The `appengine` version is a flex based custom runtime for Google Cloud.
+
+3.  via the command line.
+
+    Documents can be sent as an argument, e.g.
+
+        $ docd -input document.pdf
+
+### Optional flags
+
+- `addr` - the bind address for the HTTP server, default is ":8888"
+- `log-level`
+  - 0: errors & critical info
+  - 1: inclues 0 and logs each request as well
+  - 2: include 1 and logs the response payloads
+- `readability-length-low` - sets the readability length low if the ?readability=1 parameter is set
+- `readability-length-high` - sets the readability length high if the ?readability=1 parameter is set
+- `readability-stopwords-low` - sets the readability stopwords low if the ?readability=1 parameter is set
+- `readability-stopwords-high` - sets the readability stopwords high if the ?readability=1 parameter is set
+- `readability-max-link-density` - sets the readability max link density if the ?readability=1 parameter is set
+- `readability-max-heading-distance` - sets the readability max heading distance if the ?readability=1 parameter is set
+- `readability-use-classes` - comma separated list of readability classes to use if the ?readability=1 parameter is set
+
+### How to start the service
+
+    $ # This will only log errors and critical info
+    $ docd -log-level 0
+
+    $ # This will run on port 8000 and log each request
+    $ docd -addr :8000 -log-level 1
+
+## Example usage (code)
+
+Some basic code is shown below, but normally you would accept the file by HTTP or open it from the file system.
+
+This should be enough to get you started though.
+
+### Use case 1: run locally
+
+> Note: this assumes you have the [dependencies](#dependencies) installed.
+
+```go
+package main
+
+import (
+	"fmt"
+	"log"
+
+	"code.sajari.com/docconv"
+)
+
+func main() {
+	res, err := docconv.ConvertPath("your-file.pdf")
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println(res)
+}
+```
+
+### Use case 2: request over the network
+
+```go
+package main
+
+import (
+	"fmt"
+	"log"
+
+	"code.sajari.com/docconv/client"
+)
+
+func main() {
+	// Create a new client, using the default endpoint (localhost:8888)
+	c := client.New()
+
+	res, err := client.ConvertPath(c, "your-file.pdf")
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println(res)
+}
+```
--- a/vendor/code.sajari.com/docconv/doc.go
+++ b/vendor/code.sajari.com/docconv/doc.go
@@ -0,0 +1,94 @@
+package docconv
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+)
+
+// ConvertDoc converts an MS Word .doc to text.
+func ConvertDoc(r io.Reader) (string, map[string]string, error) {
+	f, err := NewLocalFile(r)
+	if err != nil {
+		return "", nil, fmt.Errorf("error creating local file: %v", err)
+	}
+	defer f.Done()
+
+	// Meta data
+	mc := make(chan map[string]string, 1)
+	go func() {
+		meta := make(map[string]string)
+		metaStr, err := exec.Command("wvSummary", f.Name()).Output()
+		if err != nil {
+			// TODO: Remove this.
+			log.Println("wvSummary:", err)
+		}
+
+		// Parse meta output
+		for _, line := range strings.Split(string(metaStr), "\n") {
+			if parts := strings.SplitN(line, "=", 2); len(parts) > 1 {
+				meta[strings.TrimSpace(parts[0])] = strings.TrimSpace(parts[1])
+			}
+		}
+
+		// Convert parsed meta
+		if tmp, ok := meta["Last Modified"]; ok {
+			if t, err := time.Parse(time.RFC3339, tmp); err == nil {
+				meta["ModifiedDate"] = fmt.Sprintf("%d", t.Unix())
+			}
+		}
+		if tmp, ok := meta["Created"]; ok {
+			if t, err := time.Parse(time.RFC3339, tmp); err == nil {
+				meta["CreatedDate"] = fmt.Sprintf("%d", t.Unix())
+			}
+		}
+
+		mc <- meta
+	}()
+
+	// Document body
+	bc := make(chan string, 1)
+	go func() {
+
+		// Save output to a file
+		outputFile, err := ioutil.TempFile("/tmp", "sajari-convert-")
+		if err != nil {
+			// TODO: Remove this.
+			log.Println("TempFile Out:", err)
+			return
+		}
+		defer os.Remove(outputFile.Name())
+
+		err = exec.Command("wvText", f.Name(), outputFile.Name()).Run()
+		if err != nil {
+			// TODO: Remove this.
+			log.Println("wvText:", err)
+		}
+
+		var buf bytes.Buffer
+		_, err = buf.ReadFrom(outputFile)
+		if err != nil {
+			// TODO: Remove this.
+			log.Println("wvText:", err)
+		}
+
+		bc <- buf.String()
+	}()
+
+	// TODO: Should errors in either of the above Goroutines stop things from progressing?
+	body := <-bc
+	meta := <-mc
+
+	// TODO: Check for errors instead of len(body) == 0?
+	if len(body) == 0 {
+		f.Seek(0, 0)
+		return ConvertDocx(f)
+	}
+	return body, meta, nil
+}
--- a/vendor/code.sajari.com/docconv/docconv.go
+++ b/vendor/code.sajari.com/docconv/docconv.go
@@ -0,0 +1,145 @@
+package docconv // import "code.sajari.com/docconv"
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"strings"
+	"time"
+)
+
+// Response payload sent back to the requestor
+type Response struct {
+	Body  string            `json:"body"`
+	Meta  map[string]string `json:"meta"`
+	MSecs uint32            `json:"msecs"`
+	Error string            `json:"error"`
+}
+
+// MimeTypeByExtension returns a mimetype for the given extension, or
+// application/octet-stream if none can be determined.
+func MimeTypeByExtension(filename string) string {
+	switch strings.ToLower(path.Ext(filename)) {
+	case ".doc":
+		return "application/msword"
+	case ".docx":
+		return "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+	case ".odt":
+		return "application/vnd.oasis.opendocument.text"
+	case ".pages":
+		return "application/vnd.apple.pages"
+	case ".pdf":
+		return "application/pdf"
+	case ".pptx":
+		return "application/vnd.openxmlformats-officedocument.presentationml.presentation"
+	case ".rtf":
+		return "application/rtf"
+	case ".xml":
+		return "text/xml"
+	case ".xhtml", ".html", ".htm":
+		return "text/html"
+	case ".jpg", ".jpeg", ".jpe", ".jfif", ".jfif-tbnl":
+		return "image/jpeg"
+	case ".png":
+		return "image/png"
+	case ".tif":
+		return "image/tif"
+	case ".tiff":
+		return "image/tiff"
+	case ".txt":
+		return "text/plain"
+	}
+	return "application/octet-stream"
+}
+
+// Convert a file to plain text.
+func Convert(r io.Reader, mimeType string, readability bool) (*Response, error) {
+	start := time.Now()
+
+	var body string
+	var meta map[string]string
+	var err error
+	switch mimeType {
+	case "application/msword", "application/vnd.ms-word":
+		body, meta, err = ConvertDoc(r)
+
+	case "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
+		body, meta, err = ConvertDocx(r)
+
+	case "application/vnd.openxmlformats-officedocument.presentationml.presentation":
+		body, meta, err = ConvertPptx(r)
+
+	case "application/vnd.oasis.opendocument.text":
+		body, meta, err = ConvertODT(r)
+
+	case "application/vnd.apple.pages", "application/x-iwork-pages-sffpages":
+		body, meta, err = ConvertPages(r)
+
+	case "application/pdf":
+		body, meta, err = ConvertPDF(r)
+
+	case "application/rtf", "application/x-rtf", "text/rtf", "text/richtext":
+		body, meta, err = ConvertRTF(r)
+
+	case "text/html":
+		body, meta, err = ConvertHTML(r, readability)
+
+	case "text/url":
+		body, meta, err = ConvertURL(r, readability)
+
+	case "text/xml", "application/xml":
+		body, meta, err = ConvertXML(r)
+
+	case "image/jpeg", "image/png", "image/tif", "image/tiff":
+		body, meta, err = ConvertImage(r)
+
+	case "text/plain":
+		var b []byte
+		b, err = ioutil.ReadAll(r)
+		body = string(b)
+	}
+
+	if err != nil {
+		return nil, fmt.Errorf("error converting data: %v", err)
+	}
+
+	return &Response{
+		Body:  strings.TrimSpace(body),
+		Meta:  meta,
+		MSecs: uint32(time.Since(start) / time.Millisecond),
+	}, nil
+}
+
+// ConvertPath converts a local path to text.
+func ConvertPath(path string) (*Response, error) {
+	mimeType := MimeTypeByExtension(path)
+
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	return Convert(f, mimeType, true)
+}
+
+// ConvertPathReadability converts a local path to text, with the given readability
+// option.
+func ConvertPathReadability(path string, readability bool) ([]byte, error) {
+	mimeType := MimeTypeByExtension(path)
+
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	data, err := Convert(f, mimeType, readability)
+	if err != nil {
+		return nil, err
+	}
+	return json.Marshal(data)
+}
--- a/vendor/code.sajari.com/docconv/docx.go
+++ b/vendor/code.sajari.com/docconv/docx.go
@@ -0,0 +1,155 @@
+package docconv
+
+import (
+	"archive/zip"
+	"bytes"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"time"
+)
+
+type typeOverride struct {
+	XMLName     xml.Name `xml:"Override"`
+	ContentType string   `xml:"ContentType,attr"`
+	PartName    string   `xml:"PartName,attr"`
+}
+
+type contentTypeDefinition struct {
+	XMLName   xml.Name       `xml:"Types"`
+	Overrides []typeOverride `xml:"Override"`
+}
+
+// ConvertDocx converts an MS Word docx file to text.
+func ConvertDocx(r io.Reader) (string, map[string]string, error) {
+	var size int64
+
+	// Common case: if the reader is a file (or trivial wrapper), avoid
+	// loading it all into memory.
+	var ra io.ReaderAt
+	if f, ok := r.(interface {
+		io.ReaderAt
+		Stat() (os.FileInfo, error)
+	}); ok {
+		si, err := f.Stat()
+		if err != nil {
+			return "", nil, err
+		}
+		size = si.Size()
+		ra = f
+	} else {
+		b, err := ioutil.ReadAll(r)
+		if err != nil {
+			return "", nil, nil
+		}
+		size = int64(len(b))
+		ra = bytes.NewReader(b)
+	}
+
+	zr, err := zip.NewReader(ra, size)
+	if err != nil {
+		return "", nil, fmt.Errorf("error unzipping data: %v", err)
+	}
+
+	zipFiles := mapZipFiles(zr.File)
+
+	contentTypeDefinition, err := getContentTypeDefinition(zipFiles["[Content_Types].xml"])
+	if err != nil {
+		return "", nil, err
+	}
+
+	meta := make(map[string]string)
+	var textHeader, textBody, textFooter string
+	for _, override := range contentTypeDefinition.Overrides {
+		f := zipFiles[override.PartName]
+
+		switch {
+		case override.ContentType == "application/vnd.openxmlformats-package.core-properties+xml":
+			rc, err := f.Open()
+			if err != nil {
+				return "", nil, fmt.Errorf("error opening '%v' from archive: %v", f.Name, err)
+			}
+			defer rc.Close()
+
+			meta, err = XMLToMap(rc)
+			if err != nil {
+				return "", nil, fmt.Errorf("error parsing '%v': %v", f.Name, err)
+			}
+
+			if tmp, ok := meta["modified"]; ok {
+				if t, err := time.Parse(time.RFC3339, tmp); err == nil {
+					meta["ModifiedDate"] = fmt.Sprintf("%d", t.Unix())
+				}
+			}
+			if tmp, ok := meta["created"]; ok {
+				if t, err := time.Parse(time.RFC3339, tmp); err == nil {
+					meta["CreatedDate"] = fmt.Sprintf("%d", t.Unix())
+				}
+			}
+		case override.ContentType == "application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml":
+			body, err := parseDocxText(f)
+			if err != nil {
+				return "", nil, err
+			}
+			textBody += body + "\n"
+		case override.ContentType == "application/vnd.openxmlformats-officedocument.wordprocessingml.footer+xml":
+			footer, err := parseDocxText(f)
+			if err != nil {
+				return "", nil, err
+			}
+			textFooter += footer + "\n"
+		case override.ContentType == "application/vnd.openxmlformats-officedocument.wordprocessingml.header+xml":
+			header, err := parseDocxText(f)
+			if err != nil {
+				return "", nil, err
+			}
+			textHeader += header + "\n"
+		}
+
+	}
+	return textHeader + "\n" + textBody + "\n" + textFooter, meta, nil
+}
+
+func getContentTypeDefinition(zf *zip.File) (*contentTypeDefinition, error) {
+	f, err := zf.Open()
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	x := &contentTypeDefinition{}
+	if err := xml.NewDecoder(f).Decode(x); err != nil {
+		return nil, err
+	}
+	return x, nil
+}
+
+func mapZipFiles(files []*zip.File) map[string]*zip.File {
+	filesMap := make(map[string]*zip.File, 2*len(files))
+	for _, f := range files {
+		filesMap[f.Name] = f
+		filesMap["/"+f.Name] = f
+	}
+	return filesMap
+}
+
+func parseDocxText(f *zip.File) (string, error) {
+	r, err := f.Open()
+	if err != nil {
+		return "", fmt.Errorf("error opening '%v' from archive: %v", f.Name, err)
+	}
+	defer r.Close()
+
+	text, err := DocxXMLToText(r)
+	if err != nil {
+		return "", fmt.Errorf("error parsing '%v': %v", f.Name, err)
+	}
+	return text, nil
+}
+
+// DocxXMLToText converts Docx XML into plain text.
+func DocxXMLToText(r io.Reader) (string, error) {
+	return XMLToText(r, []string{"br", "p", "tab"}, []string{"instrText", "script"}, true)
+}
--- a/vendor/code.sajari.com/docconv/go.mod
+++ b/vendor/code.sajari.com/docconv/go.mod
@@ -0,0 +1,21 @@
+module code.sajari.com/docconv
+
+go 1.14
+
+require (
+	github.com/JalfResi/justext v0.0.0-20170829062021-c0282dea7198
+	github.com/PuerkitoBio/goquery v1.5.1 // indirect
+	github.com/advancedlogic/GoOse v0.0.0-20191112112754-e742535969c1
+	github.com/andybalholm/cascadia v1.2.0 // indirect
+	github.com/araddon/dateparse v0.0.0-20200409225146-d820a6159ab1 // indirect
+	github.com/go-resty/resty/v2 v2.3.0 // indirect
+	github.com/golang/protobuf v1.4.2
+	github.com/jaytaylor/html2text v0.0.0-20200412013138-3577fbdbcff7 // indirect
+	github.com/levigross/exp-html v0.0.0-20120902181939-8df60c69a8f5 // indirect
+	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/olekukonko/tablewriter v0.0.4 // indirect
+	github.com/otiai10/gosseract/v2 v2.2.4
+	github.com/pkg/errors v0.9.1 // indirect
+	golang.org/x/net v0.0.0-20200602114024-627f9648deb9
+	golang.org/x/text v0.3.2 // indirect
+)
--- a/vendor/code.sajari.com/docconv/go.sum
+++ b/vendor/code.sajari.com/docconv/go.sum
@@ -0,0 +1,99 @@
+github.com/JalfResi/justext v0.0.0-20170829062021-c0282dea7198 h1:8P+AjBhGByCuCX2zTkAf6UY+dj0JczX+t6cSdCSyvfw=
+github.com/JalfResi/justext v0.0.0-20170829062021-c0282dea7198/go.mod h1:0SURuH1rsE8aVWvutuMZghRNrNrYEUzibzJfhEYR8L0=
+github.com/PuerkitoBio/goquery v1.4.1 h1:smcIRGdYm/w7JSbcdeLHEMzxmsBQvl8lhf0dSw2nzMI=
+github.com/PuerkitoBio/goquery v1.4.1/go.mod h1:T9ezsOHcCrDCgA8aF1Cqr3sSYbO/xgdy8/R/XiIMAhA=
+github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
+github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
+github.com/advancedlogic/GoOse v0.0.0-20191112112754-e742535969c1 h1:d0Ct1dZwgwMO0Llf81Eu+Lyj6kwqXdqHP/WsSkEria0=
+github.com/advancedlogic/GoOse v0.0.0-20191112112754-e742535969c1/go.mod h1:f3HCSN1fBWjcpGtXyM119MJgeQl838v6so/PQOqvE1w=
+github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
+github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/andybalholm/cascadia v1.2.0 h1:vuRCkM5Ozh/BfmsaTm26kbjm0mIOM3yS5Ek/F5h18aE=
+github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY=
+github.com/araddon/dateparse v0.0.0-20180729174819-cfd92a431d0e h1:s05JG2GwtJMHaPcXDpo4V35TFgyYZzNsmBlSkHPEbeg=
+github.com/araddon/dateparse v0.0.0-20180729174819-cfd92a431d0e/go.mod h1:SLqhdZcd+dF3TEVL2RMoob5bBP5R1P1qkox+HtCBgGI=
+github.com/araddon/dateparse v0.0.0-20200409225146-d820a6159ab1 h1:TEBmxO80TM04L8IuMWk77SGL1HomBmKTdzdJLLWznxI=
+github.com/araddon/dateparse v0.0.0-20200409225146-d820a6159ab1/go.mod h1:SLqhdZcd+dF3TEVL2RMoob5bBP5R1P1qkox+HtCBgGI=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fatih/set v0.2.1 h1:nn2CaJyknWE/6txyUDGwysr3G5QC6xWB/PtVjPBbeaA=
+github.com/fatih/set v0.2.1/go.mod h1:+RKtMCH+favT2+3YecHGxcc0b4KyVWA1QWWJUs4E0CI=
+github.com/gigawattio/window v0.0.0-20180317192513-0f5467e35573 h1:u8AQ9bPa9oC+8/A/jlWouakhIvkFfuxgIIRjiy8av7I=
+github.com/gigawattio/window v0.0.0-20180317192513-0f5467e35573/go.mod h1:eBvb3i++NHDH4Ugo9qCvMw8t0mTSctaEa5blJbWcNxs=
+github.com/go-resty/resty/v2 v2.0.0 h1:9Nq/U+V4xsoDnDa/iTrABDWUCuk3Ne92XFHPe6dKWUc=
+github.com/go-resty/resty/v2 v2.0.0/go.mod h1:dZGr0i9PLlaaTD4H/hoZIDjQ+r6xq8mgbRzHZf7f2J8=
+github.com/go-resty/resty/v2 v2.3.0 h1:JOOeAvjSlapTT92p8xiS19Zxev1neGikoHsXJeOq8So=
+github.com/go-resty/resty/v2 v2.3.0/go.mod h1:UpN9CgLZNsv4e9XG50UU8xdI0F43UQ4HmxLBDwaroHU=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0 h1:xqgexXAGQgY3HAjNPSaCqn5Aahbo5TKsmhp8VRfr1iQ=
+github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk=
+github.com/jaytaylor/html2text v0.0.0-20200412013138-3577fbdbcff7 h1:g0fAGBisHaEQ0TRq1iBvemFRf+8AEWEmBESSiWB3Vsc=
+github.com/jaytaylor/html2text v0.0.0-20200412013138-3577fbdbcff7/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk=
+github.com/levigross/exp-html v0.0.0-20120902181939-8df60c69a8f5 h1:W7p+m/AECTL3s/YR5RpQ4hz5SjNeKzZBl1q36ws12s0=
+github.com/levigross/exp-html v0.0.0-20120902181939-8df60c69a8f5/go.mod h1:QMe2wuKJ0o7zIVE8AqiT8rd8epmm6WDIZ2wyuBqYPzM=
+github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4=
+github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/olekukonko/tablewriter v0.0.0-20180506121414-d4647c9c7a84 h1:fiKJgB4JDUd43CApkmCeTSQlWjtTtABrU2qsgbuP0BI=
+github.com/olekukonko/tablewriter v0.0.0-20180506121414-d4647c9c7a84/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
+github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8=
+github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA=
+github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95 h1:+OLn68pqasWca0z5ryit9KGfp3sUsW4Lqg32iRMJyzs=
+github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
+github.com/otiai10/gosseract/v2 v2.2.4 h1:h/PV+oJqke8q2Ccw9bjpMBWfd7N2vtGDCUcihZj3nRo=
+github.com/otiai10/gosseract/v2 v2.2.4/go.mod h1:ahOp/kHojnOMGv1RaUnR0jwY5JVa6BYKhYAS8nbMLSo=
+github.com/otiai10/mint v1.3.0 h1:Ady6MKVezQwHBkGzLFbrsywyp09Ah7rkmfjV3Bcr5uc=
+github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/simplereach/timeutils v1.2.0/go.mod h1:VVbQDfN/FHRZa1LSqcwo4kNZ62OOyqLLGQKYB3pB0Q8=
+github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo=
+github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM=
+golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
--- a/vendor/code.sajari.com/docconv/html.go
+++ b/vendor/code.sajari.com/docconv/html.go
@@ -0,0 +1,191 @@
+// +build !appengine
+
+package docconv
+
+import (
+	"bytes"
+	"io"
+	"log"
+	"strings"
+
+	"golang.org/x/net/html"
+
+	"github.com/JalfResi/justext"
+)
+
+// ConvertHTML converts HTML into text.
+func ConvertHTML(r io.Reader, readability bool) (string, map[string]string, error) {
+	meta := make(map[string]string)
+
+	buf := new(bytes.Buffer)
+	_, err := buf.ReadFrom(r)
+	if err != nil {
+		return "", nil, err
+	}
+
+	cleanXML, err := Tidy(buf, false)
+	if err != nil {
+		log.Println("Tidy:", err)
+		// Tidy failed, so we now manually tokenize instead
+		clean := cleanHTML(buf, true)
+		cleanXML = []byte(clean)
+		// TODO: remove this log
+		log.Println("Cleaned HTML using Golang tokenizer")
+	}
+
+	if readability {
+		cleanXML = HTMLReadability(bytes.NewReader(cleanXML))
+	}
+	return HTMLToText(bytes.NewReader(cleanXML)), meta, nil
+}
+
+var acceptedHTMLTags = [...]string{
+	"div", "p", "br", "span", "body", "head", "html", "ul", "ol", "li", "dl", "dt", "dd", "a", "form", "article",
+	"section", "table", "tr", "td", "tbody", "thead", "th", "tfoot", "col", "colgroup", "caption", "form", "input",
+	"title", "h1", "h2", "h3", "h4", "h5", "h6", "meta", "strong", "cite", "em", "address", "abbr", "acronym",
+	"blockquote", "q", "pre", "samp", "select", "fieldset", "legend", "button", "option", "textarea", "label",
+}
+
+// Tests for known friendly HTML parameters that tidy is unlikely to choke on
+func acceptedHTMLTag(tagName string) bool {
+	for _, tag := range acceptedHTMLTags {
+		if tag == tagName {
+			return true
+		}
+	}
+	return false
+}
+
+// Removes scripts, comments, styles and parameters from HTML.
+// Also removes made up tags, e.g. <fb:like>
+// Can keep head elements or not. Typically not much in there.
+func cleanHTML(r io.Reader, all bool) string {
+	output := ""
+	if !all {
+		output = "<html><head></head>"
+	}
+	mainSection := false
+	junkSection := false
+
+	d := html.NewTokenizer(r)
+	for {
+		// token type
+		tokenType := d.Next()
+		if tokenType == html.ErrorToken {
+			return output
+		}
+		token := d.Token()
+
+		switch tokenType {
+		case html.StartTagToken: // <tag>
+			if token.Data == "body" || (token.Data == "html" && all) {
+				mainSection = true
+			}
+			if !acceptedHTMLTag(token.Data) {
+				junkSection = true
+			}
+
+			if !junkSection && mainSection {
+				output += "<" + token.Data + ">"
+			}
+
+		case html.TextToken: // text between start and end tag
+			if !junkSection && mainSection {
+				output += token.Data
+			}
+
+		case html.EndTagToken: // </tag>
+			if !junkSection && mainSection {
+				output += "</" + token.Data + ">"
+			}
+			if !acceptedHTMLTag(token.Data) {
+				junkSection = false
+			}
+
+		case html.SelfClosingTagToken: // <tag/>
+			if !junkSection && mainSection {
+				output += "<" + token.Data + " />" // TODO: Can probably keep attributes from the meta tags
+			}
+		}
+	}
+}
+
+// HTMLReadabilityOptions is a type which defines parameters that are passed to the justext package.
+// TODO: Improve this!
+type HTMLReadabilityOptions struct {
+	LengthLow             int
+	LengthHigh            int
+	StopwordsLow          float64
+	StopwordsHigh         float64
+	MaxLinkDensity        float64
+	MaxHeadingDistance    int
+	ReadabilityUseClasses string
+}
+
+// HTMLReadabilityOptionsValues are the global settings used for HTMLReadability.
+// TODO: Remove this from global state.
+var HTMLReadabilityOptionsValues HTMLReadabilityOptions
+
+// HTMLReadability extracts the readable text in an HTML document
+func HTMLReadability(r io.Reader) []byte {
+	jr := justext.NewReader(r)
+
+	// TODO: Improve this!
+	jr.Stoplist = readabilityStopList
+	jr.LengthLow = HTMLReadabilityOptionsValues.LengthLow
+	jr.LengthHigh = HTMLReadabilityOptionsValues.LengthHigh
+	jr.StopwordsLow = HTMLReadabilityOptionsValues.StopwordsLow
+	jr.StopwordsHigh = HTMLReadabilityOptionsValues.StopwordsHigh
+	jr.MaxLinkDensity = HTMLReadabilityOptionsValues.MaxLinkDensity
+	jr.MaxHeadingDistance = HTMLReadabilityOptionsValues.MaxHeadingDistance
+
+	paragraphSet, err := jr.ReadAll()
+	if err != nil {
+		log.Println("Justext:", err)
+		return nil
+	}
+
+	useClasses := strings.SplitN(HTMLReadabilityOptionsValues.ReadabilityUseClasses, ",", 10)
+
+	output := ""
+	for _, paragraph := range paragraphSet {
+		for _, class := range useClasses {
+			if paragraph.CfClass == class {
+				output += paragraph.Text + "\n"
+			}
+		}
+	}
+
+	return []byte(output)
+}
+
+// HTMLToText converts HTML to plain text.
+func HTMLToText(input io.Reader) string {
+	text, _ := XMLToText(input, []string{"br", "p", "h1", "h2", "h3", "h4"}, []string{}, false)
+	return text
+}
+
+var readabilityStopList = map[string]bool{"and": true, "the": true, "a": true, "about": true, "above": true, "across": true, "after": true, "afterwards": true, "again": true, "against": true, "all": true, "almost": true, "alone": true,
+	"along": true, "already": true, "also": true, "although": true, "always": true, "am": true, "among": true, "amongst": true, "amoungst": true, "amount": true, "an": true, "another": true, "any": true,
+	"anyhow": true, "anyone": true, "anything": true, "anyway": true, "anywhere": true, "are": true, "around": true, "as": true, "at": true, "back": true, "be": true, "became": true, "because": true,
+	"become": true, "becomes": true, "becoming": true, "been": true, "before": true, "beforehand": true, "behind": true, "being": true, "below": true, "beside": true, "besides": true, "between": true,
+	"beyond": true, "both": true, "bottom": true, "but": true, "by": true, "can": true, "cannot": true, "cant": true, "co": true, "con": true, "could": true, "couldnt": true, "cry": true,
+	"de": true, "describe": true, "detail": true, "do": true, "done": true, "down": true, "due": true, "during": true, "each": true, "eg": true, "eight": true, "either": true, "eleven": true, "else": true,
+	"elsewhere": true, "empty": true, "enough": true, "etc": true, "even": true, "ever": true, "every": true, "everyone": true, "everything": true, "everywhere": true, "except": true, "few": true,
+	"fifteen": true, "fify": true, "fill": true, "find": true, "fire": true, "first": true, "five": true, "for": true, "former": true, "formerly": true, "forty": true, "found": true, "four": true, "from": true,
+	"front": true, "full": true, "further": true, "get": true, "give": true, "go": true, "had": true, "has": true, "hasnt": true, "have": true, "he": true, "hence": true, "her": true, "here": true, "hereafter": true,
+	"hereby": true, "herein": true, "hereupon": true, "hers": true, "herself": true, "him": true, "himself": true, "his": true, "how": true, "however": true, "hundred": true, "ie": true, "if": true, "in": true,
+	"inc": true, "indeed": true, "interest": true, "into": true, "is": true, "it": true, "its": true, "itself": true, "keep": true, "last": true, "latter": true, "latterly": true, "least": true, "less": true,
+	"ltd": true, "made": true, "many": true, "may": true, "me": true, "meanwhile": true, "might": true, "mill": true, "mine": true, "more": true, "moreover": true, "most": true, "mostly": true, "move": true,
+	"much": true, "must": true, "my": true, "myself": true, "name": true, "namely": true, "neither": true, "never": true, "nevertheless": true, "next": true, "nine": true, "no": true, "nobody": true,
+	"none": true, "noone": true, "nor": true, "not": true, "nothing": true, "now": true, "nowhere": true, "of": true, "off": true, "often": true, "on": true, "once": true, "one": true, "only": true, "onto": true,
+	"or": true, "other": true, "others": true, "otherwise": true, "our": true, "ours": true, "ourselves": true, "out": true, "over": true, "own": true, "part": true, "per": true, "perhaps": true,
+	"please": true, "put": true, "rather": true, "re": true, "same": true, "see": true, "seem": true, "seemed": true, "seeming": true, "seems": true, "serious": true, "several": true, "she": true,
+	"should": true, "show": true, "side": true, "since": true, "sincere": true, "six": true, "sixty": true, "so": true, "some": true, "somehow": true, "someone": true, "something": true, "sometime": true,
+	"sometimes": true, "somewhere": true, "still": true, "such": true, "take": true, "ten": true, "than": true, "that": true, "their": true, "them": true, "themselves": true,
+	"then": true, "thence": true, "there": true, "thereafter": true, "thereby": true, "therefore": true, "therein": true, "thereupon": true, "these": true, "they": true, "thickv": true, "thin": true,
+	"third": true, "this": true, "those": true, "though": true, "three": true, "through": true, "throughout": true, "thru": true, "thus": true, "to": true, "together": true, "too": true, "top": true,
+	"toward": true, "towards": true, "twelve": true, "twenty": true, "two": true, "un": true, "under": true, "until": true, "up": true, "upon": true, "us": true, "very": true, "via": true, "was": true, "we": true,
+	"well": true, "were": true, "what": true, "whatever": true, "when": true, "whence": true, "whenever": true, "where": true, "whereafter": true, "whereas": true, "whereby": true, "wherein": true,
+	"whereupon": true, "wherever": true, "whether": true, "which": true, "while": true, "whither": true, "who": true, "whoever": true, "whole": true, "whom": true, "whose": true, "why": true, "will": true,
+	"with": true, "within": true, "without": true, "would": true, "yet": true, "you": true, "your": true, "youre": true, "yours": true, "yourself": true, "yourselves": true, "www": true, "com": true, "http": true}
--- a/vendor/code.sajari.com/docconv/html_appengine.go
+++ b/vendor/code.sajari.com/docconv/html_appengine.go
@@ -0,0 +1,18 @@
+// +build appengine
+
+package docconv
+
+import (
+	"io"
+	"io/ioutil"
+	"log"
+)
+
+func HTMLReadability(r io.Reader) []byte {
+	b, err := ioutil.ReadAll(r)
+	if err != nil {
+		log.Printf("HTMLReadability: %v", err)
+		return nil
+	}
+	return b
+}
--- a/vendor/code.sajari.com/docconv/iWork/TSPArchiveMessages.pb.go
+++ b/vendor/code.sajari.com/docconv/iWork/TSPArchiveMessages.pb.go
@@ -0,0 +1,587 @@
+// Code generated by protoc-gen-go.
+// source: TSPArchiveMessages.proto
+// DO NOT EDIT!
+
+/*
+Package TSP is a generated protocol buffer package.
+
+It is generated from these files:
+	TSPArchiveMessages.proto
+	TSPDatabaseMessages.proto
+	TSPMessages.proto
+
+It has these top-level messages:
+	ArchiveInfo
+	MessageInfo
+	FieldInfo
+	FieldPath
+	ComponentInfo
+	ComponentExternalReference
+	ComponentDataReference
+	PackageMetadata
+	PasteboardMetadata
+	DataInfo
+	ViewStateMetadata
+*/
+package TSP
+
+import proto "github.com/golang/protobuf/proto"
+import math "math"
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = math.Inf
+
+type FieldInfo_Type int32
+
+const (
+	FieldInfo_Value           FieldInfo_Type = 0
+	FieldInfo_ObjectReference FieldInfo_Type = 1
+	FieldInfo_DataReference   FieldInfo_Type = 2
+	FieldInfo_Message         FieldInfo_Type = 3
+)
+
+var FieldInfo_Type_name = map[int32]string{
+	0: "Value",
+	1: "ObjectReference",
+	2: "DataReference",
+	3: "Message",
+}
+var FieldInfo_Type_value = map[string]int32{
+	"Value":           0,
+	"ObjectReference": 1,
+	"DataReference":   2,
+	"Message":         3,
+}
+
+func (x FieldInfo_Type) Enum() *FieldInfo_Type {
+	p := new(FieldInfo_Type)
+	*p = x
+	return p
+}
+func (x FieldInfo_Type) String() string {
+	return proto.EnumName(FieldInfo_Type_name, int32(x))
+}
+func (x *FieldInfo_Type) UnmarshalJSON(data []byte) error {
+	value, err := proto.UnmarshalJSONEnum(FieldInfo_Type_value, data, "FieldInfo_Type")
+	if err != nil {
+		return err
+	}
+	*x = FieldInfo_Type(value)
+	return nil
+}
+
+type FieldInfo_Rule int32
+
+const (
+	FieldInfo_IgnoreAndDrop     FieldInfo_Rule = 0
+	FieldInfo_IgnoreAndPreserve FieldInfo_Rule = 1
+	FieldInfo_MustUnderstand    FieldInfo_Rule = 2
+	FieldInfo_NotSupported      FieldInfo_Rule = -1
+)
+
+var FieldInfo_Rule_name = map[int32]string{
+	0:  "IgnoreAndDrop",
+	1:  "IgnoreAndPreserve",
+	2:  "MustUnderstand",
+	-1: "NotSupported",
+}
+var FieldInfo_Rule_value = map[string]int32{
+	"IgnoreAndDrop":     0,
+	"IgnoreAndPreserve": 1,
+	"MustUnderstand":    2,
+	"NotSupported":      -1,
+}
+
+func (x FieldInfo_Rule) Enum() *FieldInfo_Rule {
+	p := new(FieldInfo_Rule)
+	*p = x
+	return p
+}
+func (x FieldInfo_Rule) String() string {
+	return proto.EnumName(FieldInfo_Rule_name, int32(x))
+}
+func (x *FieldInfo_Rule) UnmarshalJSON(data []byte) error {
+	value, err := proto.UnmarshalJSONEnum(FieldInfo_Rule_value, data, "FieldInfo_Rule")
+	if err != nil {
+		return err
+	}
+	*x = FieldInfo_Rule(value)
+	return nil
+}
+
+type ArchiveInfo struct {
+	Identifier       *uint64        `protobuf:"varint,1,opt,name=identifier" json:"identifier,omitempty"`
+	MessageInfos     []*MessageInfo `protobuf:"bytes,2,rep,name=message_infos" json:"message_infos,omitempty"`
+	XXX_unrecognized []byte         `json:"-"`
+}
+
+func (m *ArchiveInfo) Reset()         { *m = ArchiveInfo{} }
+func (m *ArchiveInfo) String() string { return proto.CompactTextString(m) }
+func (*ArchiveInfo) ProtoMessage()    {}
+
+func (m *ArchiveInfo) GetIdentifier() uint64 {
+	if m != nil && m.Identifier != nil {
+		return *m.Identifier
+	}
+	return 0
+}
+
+func (m *ArchiveInfo) GetMessageInfos() []*MessageInfo {
+	if m != nil {
+		return m.MessageInfos
+	}
+	return nil
+}
+
+type MessageInfo struct {
+	Type             *uint32      `protobuf:"varint,1,req,name=type" json:"type,omitempty"`
+	Version          []uint32     `protobuf:"varint,2,rep,packed,name=version" json:"version,omitempty"`
+	Length           *uint32      `protobuf:"varint,3,req,name=length" json:"length,omitempty"`
+	FieldInfos       []*FieldInfo `protobuf:"bytes,4,rep,name=field_infos" json:"field_infos,omitempty"`
+	ObjectReferences []uint64     `protobuf:"varint,5,rep,packed,name=object_references" json:"object_references,omitempty"`
+	DataReferences   []uint64     `protobuf:"varint,6,rep,packed,name=data_references" json:"data_references,omitempty"`
+	XXX_unrecognized []byte       `json:"-"`
+}
+
+func (m *MessageInfo) Reset()         { *m = MessageInfo{} }
+func (m *MessageInfo) String() string { return proto.CompactTextString(m) }
+func (*MessageInfo) ProtoMessage()    {}
+
+func (m *MessageInfo) GetType() uint32 {
+	if m != nil && m.Type != nil {
+		return *m.Type
+	}
+	return 0
+}
+
+func (m *MessageInfo) GetVersion() []uint32 {
+	if m != nil {
+		return m.Version
+	}
+	return nil
+}
+
+func (m *MessageInfo) GetLength() uint32 {
+	if m != nil && m.Length != nil {
+		return *m.Length
+	}
+	return 0
+}
+
+func (m *MessageInfo) GetFieldInfos() []*FieldInfo {
+	if m != nil {
+		return m.FieldInfos
+	}
+	return nil
+}
+
+func (m *MessageInfo) GetObjectReferences() []uint64 {
+	if m != nil {
+		return m.ObjectReferences
+	}
+	return nil
+}
+
+func (m *MessageInfo) GetDataReferences() []uint64 {
+	if m != nil {
+		return m.DataReferences
+	}
+	return nil
+}
+
+type FieldInfo struct {
+	Path             *FieldPath      `protobuf:"bytes,1,req,name=path" json:"path,omitempty"`
+	Type             *FieldInfo_Type `protobuf:"varint,2,opt,name=type,enum=TSP.FieldInfo_Type,def=0" json:"type,omitempty"`
+	Rule             *FieldInfo_Rule `protobuf:"varint,3,opt,name=rule,enum=TSP.FieldInfo_Rule,def=0" json:"rule,omitempty"`
+	ObjectReferences []uint64        `protobuf:"varint,4,rep,packed,name=object_references" json:"object_references,omitempty"`
+	DataReferences   []uint64        `protobuf:"varint,5,rep,packed,name=data_references" json:"data_references,omitempty"`
+	XXX_unrecognized []byte          `json:"-"`
+}
+
+func (m *FieldInfo) Reset()         { *m = FieldInfo{} }
+func (m *FieldInfo) String() string { return proto.CompactTextString(m) }
+func (*FieldInfo) ProtoMessage()    {}
+
+const Default_FieldInfo_Type FieldInfo_Type = FieldInfo_Value
+const Default_FieldInfo_Rule FieldInfo_Rule = FieldInfo_IgnoreAndDrop
+
+func (m *FieldInfo) GetPath() *FieldPath {
+	if m != nil {
+		return m.Path
+	}
+	return nil
+}
+
+func (m *FieldInfo) GetType() FieldInfo_Type {
+	if m != nil && m.Type != nil {
+		return *m.Type
+	}
+	return Default_FieldInfo_Type
+}
+
+func (m *FieldInfo) GetRule() FieldInfo_Rule {
+	if m != nil && m.Rule != nil {
+		return *m.Rule
+	}
+	return Default_FieldInfo_Rule
+}
+
+func (m *FieldInfo) GetObjectReferences() []uint64 {
+	if m != nil {
+		return m.ObjectReferences
+	}
+	return nil
+}
+
+func (m *FieldInfo) GetDataReferences() []uint64 {
+	if m != nil {
+		return m.DataReferences
+	}
+	return nil
+}
+
+type FieldPath struct {
+	Path             []uint32 `protobuf:"varint,1,rep,packed,name=path" json:"path,omitempty"`
+	XXX_unrecognized []byte   `json:"-"`
+}
+
+func (m *FieldPath) Reset()         { *m = FieldPath{} }
+func (m *FieldPath) String() string { return proto.CompactTextString(m) }
+func (*FieldPath) ProtoMessage()    {}
+
+func (m *FieldPath) GetPath() []uint32 {
+	if m != nil {
+		return m.Path
+	}
+	return nil
+}
+
+type ComponentInfo struct {
+	Identifier                               *uint64                       `protobuf:"varint,1,req,name=identifier" json:"identifier,omitempty"`
+	PreferredLocator                         *string                       `protobuf:"bytes,2,req,name=preferred_locator" json:"preferred_locator,omitempty"`
+	Locator                                  *string                       `protobuf:"bytes,3,opt,name=locator" json:"locator,omitempty"`
+	ReadVersion                              []uint32                      `protobuf:"varint,4,rep,packed,name=read_version" json:"read_version,omitempty"`
+	WriteVersion                             []uint32                      `protobuf:"varint,5,rep,packed,name=write_version" json:"write_version,omitempty"`
+	ExternalReferences                       []*ComponentExternalReference `protobuf:"bytes,6,rep,name=external_references" json:"external_references,omitempty"`
+	DataReferences                           []*ComponentDataReference     `protobuf:"bytes,7,rep,name=data_references" json:"data_references,omitempty"`
+	AllowsDuplicatesOutsideOfDocumentPackage *bool                         `protobuf:"varint,8,opt,name=allows_duplicates_outside_of_document_package,def=0" json:"allows_duplicates_outside_of_document_package,omitempty"`
+	DirtiesDocumentPackage                   *bool                         `protobuf:"varint,9,opt,name=dirties_document_package,def=1" json:"dirties_document_package,omitempty"`
+	IsStoredOutsideObjectArchive             *bool                         `protobuf:"varint,10,opt,name=is_stored_outside_object_archive,def=0" json:"is_stored_outside_object_archive,omitempty"`
+	XXX_unrecognized                         []byte                        `json:"-"`
+}
+
+func (m *ComponentInfo) Reset()         { *m = ComponentInfo{} }
+func (m *ComponentInfo) String() string { return proto.CompactTextString(m) }
+func (*ComponentInfo) ProtoMessage()    {}
+
+const Default_ComponentInfo_AllowsDuplicatesOutsideOfDocumentPackage bool = false
+const Default_ComponentInfo_DirtiesDocumentPackage bool = true
+const Default_ComponentInfo_IsStoredOutsideObjectArchive bool = false
+
+func (m *ComponentInfo) GetIdentifier() uint64 {
+	if m != nil && m.Identifier != nil {
+		return *m.Identifier
+	}
+	return 0
+}
+
+func (m *ComponentInfo) GetPreferredLocator() string {
+	if m != nil && m.PreferredLocator != nil {
+		return *m.PreferredLocator
+	}
+	return ""
+}
+
+func (m *ComponentInfo) GetLocator() string {
+	if m != nil && m.Locator != nil {
+		return *m.Locator
+	}
+	return ""
+}
+
+func (m *ComponentInfo) GetReadVersion() []uint32 {
+	if m != nil {
+		return m.ReadVersion
+	}
+	return nil
+}
+
+func (m *ComponentInfo) GetWriteVersion() []uint32 {
+	if m != nil {
+		return m.WriteVersion
+	}
+	return nil
+}
+
+func (m *ComponentInfo) GetExternalReferences() []*ComponentExternalReference {
+	if m != nil {
+		return m.ExternalReferences
+	}
+	return nil
+}
+
+func (m *ComponentInfo) GetDataReferences() []*ComponentDataReference {
+	if m != nil {
+		return m.DataReferences
+	}
+	return nil
+}
+
+func (m *ComponentInfo) GetAllowsDuplicatesOutsideOfDocumentPackage() bool {
+	if m != nil && m.AllowsDuplicatesOutsideOfDocumentPackage != nil {
+		return *m.AllowsDuplicatesOutsideOfDocumentPackage
+	}
+	return Default_ComponentInfo_AllowsDuplicatesOutsideOfDocumentPackage
+}
+
+func (m *ComponentInfo) GetDirtiesDocumentPackage() bool {
+	if m != nil && m.DirtiesDocumentPackage != nil {
+		return *m.DirtiesDocumentPackage
+	}
+	return Default_ComponentInfo_DirtiesDocumentPackage
+}
+
+func (m *ComponentInfo) GetIsStoredOutsideObjectArchive() bool {
+	if m != nil && m.IsStoredOutsideObjectArchive != nil {
+		return *m.IsStoredOutsideObjectArchive
+	}
+	return Default_ComponentInfo_IsStoredOutsideObjectArchive
+}
+
+type ComponentExternalReference struct {
+	ComponentIdentifier *uint64 `protobuf:"varint,1,req,name=component_identifier" json:"component_identifier,omitempty"`
+	ObjectIdentifier    *uint64 `protobuf:"varint,2,opt,name=object_identifier" json:"object_identifier,omitempty"`
+	IsWeak              *bool   `protobuf:"varint,3,opt,name=is_weak" json:"is_weak,omitempty"`
+	XXX_unrecognized    []byte  `json:"-"`
+}
+
+func (m *ComponentExternalReference) Reset()         { *m = ComponentExternalReference{} }
+func (m *ComponentExternalReference) String() string { return proto.CompactTextString(m) }
+func (*ComponentExternalReference) ProtoMessage()    {}
+
+func (m *ComponentExternalReference) GetComponentIdentifier() uint64 {
+	if m != nil && m.ComponentIdentifier != nil {
+		return *m.ComponentIdentifier
+	}
+	return 0
+}
+
+func (m *ComponentExternalReference) GetObjectIdentifier() uint64 {
+	if m != nil && m.ObjectIdentifier != nil {
+		return *m.ObjectIdentifier
+	}
+	return 0
+}
+
+func (m *ComponentExternalReference) GetIsWeak() bool {
+	if m != nil && m.IsWeak != nil {
+		return *m.IsWeak
+	}
+	return false
+}
+
+type ComponentDataReference struct {
+	DataIdentifier   *uint64 `protobuf:"varint,1,req,name=data_identifier" json:"data_identifier,omitempty"`
+	XXX_unrecognized []byte  `json:"-"`
+}
+
+func (m *ComponentDataReference) Reset()         { *m = ComponentDataReference{} }
+func (m *ComponentDataReference) String() string { return proto.CompactTextString(m) }
+func (*ComponentDataReference) ProtoMessage()    {}
+
+func (m *ComponentDataReference) GetDataIdentifier() uint64 {
+	if m != nil && m.DataIdentifier != nil {
+		return *m.DataIdentifier
+	}
+	return 0
+}
+
+type PackageMetadata struct {
+	LastObjectIdentifier *uint64          `protobuf:"varint,1,req,name=last_object_identifier" json:"last_object_identifier,omitempty"`
+	Components           []*ComponentInfo `protobuf:"bytes,3,rep,name=components" json:"components,omitempty"`
+	Datas                []*DataInfo      `protobuf:"bytes,4,rep,name=datas" json:"datas,omitempty"`
+	ReadVersion          []uint32         `protobuf:"varint,5,rep,packed,name=read_version" json:"read_version,omitempty"`
+	WriteVersion         []uint32         `protobuf:"varint,6,rep,packed,name=write_version" json:"write_version,omitempty"`
+	XXX_unrecognized     []byte           `json:"-"`
+}
+
+func (m *PackageMetadata) Reset()         { *m = PackageMetadata{} }
+func (m *PackageMetadata) String() string { return proto.CompactTextString(m) }
+func (*PackageMetadata) ProtoMessage()    {}
+
+func (m *PackageMetadata) GetLastObjectIdentifier() uint64 {
+	if m != nil && m.LastObjectIdentifier != nil {
+		return *m.LastObjectIdentifier
+	}
+	return 0
+}
+
+func (m *PackageMetadata) GetComponents() []*ComponentInfo {
+	if m != nil {
+		return m.Components
+	}
+	return nil
+}
+
+func (m *PackageMetadata) GetDatas() []*DataInfo {
+	if m != nil {
+		return m.Datas
+	}
+	return nil
+}
+
+func (m *PackageMetadata) GetReadVersion() []uint32 {
+	if m != nil {
+		return m.ReadVersion
+	}
+	return nil
+}
+
+func (m *PackageMetadata) GetWriteVersion() []uint32 {
+	if m != nil {
+		return m.WriteVersion
+	}
+	return nil
+}
+
+type PasteboardMetadata struct {
+	Version            []uint32    `protobuf:"varint,1,rep,packed,name=version" json:"version,omitempty"`
+	AppName            *string     `protobuf:"bytes,2,req,name=app_name" json:"app_name,omitempty"`
+	Datas              []*DataInfo `protobuf:"bytes,3,rep,name=datas" json:"datas,omitempty"`
+	SourceDocumentUuid *string     `protobuf:"bytes,4,opt,name=source_document_uuid" json:"source_document_uuid,omitempty"`
+	XXX_unrecognized   []byte      `json:"-"`
+}
+
+func (m *PasteboardMetadata) Reset()         { *m = PasteboardMetadata{} }
+func (m *PasteboardMetadata) String() string { return proto.CompactTextString(m) }
+func (*PasteboardMetadata) ProtoMessage()    {}
+
+func (m *PasteboardMetadata) GetVersion() []uint32 {
+	if m != nil {
+		return m.Version
+	}
+	return nil
+}
+
+func (m *PasteboardMetadata) GetAppName() string {
+	if m != nil && m.AppName != nil {
+		return *m.AppName
+	}
+	return ""
+}
+
+func (m *PasteboardMetadata) GetDatas() []*DataInfo {
+	if m != nil {
+		return m.Datas
+	}
+	return nil
+}
+
+func (m *PasteboardMetadata) GetSourceDocumentUuid() string {
+	if m != nil && m.SourceDocumentUuid != nil {
+		return *m.SourceDocumentUuid
+	}
+	return ""
+}
+
+type DataInfo struct {
+	Identifier                 *uint64 `protobuf:"varint,1,req,name=identifier" json:"identifier,omitempty"`
+	Digest                     []byte  `protobuf:"bytes,2,req,name=digest" json:"digest,omitempty"`
+	PreferredFileName          *string `protobuf:"bytes,3,req,name=preferred_file_name" json:"preferred_file_name,omitempty"`
+	FileName                   *string `protobuf:"bytes,4,opt,name=file_name" json:"file_name,omitempty"`
+	DocumentResourceLocator    *string `protobuf:"bytes,5,opt,name=document_resource_locator" json:"document_resource_locator,omitempty"`
+	SourceBookmarkData         []byte  `protobuf:"bytes,6,opt,name=source_bookmark_data" json:"source_bookmark_data,omitempty"`
+	PasteboardExternalFilePath *string `protobuf:"bytes,99,opt,name=pasteboard_external_file_path" json:"pasteboard_external_file_path,omitempty"`
+	XXX_unrecognized           []byte  `json:"-"`
+}
+
+func (m *DataInfo) Reset()         { *m = DataInfo{} }
+func (m *DataInfo) String() string { return proto.CompactTextString(m) }
+func (*DataInfo) ProtoMessage()    {}
+
+func (m *DataInfo) GetIdentifier() uint64 {
+	if m != nil && m.Identifier != nil {
+		return *m.Identifier
+	}
+	return 0
+}
+
+func (m *DataInfo) GetDigest() []byte {
+	if m != nil {
+		return m.Digest
+	}
+	return nil
+}
+
+func (m *DataInfo) GetPreferredFileName() string {
+	if m != nil && m.PreferredFileName != nil {
+		return *m.PreferredFileName
+	}
+	return ""
+}
+
+func (m *DataInfo) GetFileName() string {
+	if m != nil && m.FileName != nil {
+		return *m.FileName
+	}
+	return ""
+}
+
+func (m *DataInfo) GetDocumentResourceLocator() string {
+	if m != nil && m.DocumentResourceLocator != nil {
+		return *m.DocumentResourceLocator
+	}
+	return ""
+}
+
+func (m *DataInfo) GetSourceBookmarkData() []byte {
+	if m != nil {
+		return m.SourceBookmarkData
+	}
+	return nil
+}
+
+func (m *DataInfo) GetPasteboardExternalFilePath() string {
+	if m != nil && m.PasteboardExternalFilePath != nil {
+		return *m.PasteboardExternalFilePath
+	}
+	return ""
+}
+
+type ViewStateMetadata struct {
+	Version             []uint32       `protobuf:"varint,1,rep,packed,name=version" json:"version,omitempty"`
+	DocumentVersionUuid *string        `protobuf:"bytes,2,req,name=document_version_uuid" json:"document_version_uuid,omitempty"`
+	Component           *ComponentInfo `protobuf:"bytes,3,req,name=component" json:"component,omitempty"`
+	XXX_unrecognized    []byte         `json:"-"`
+}
+
+func (m *ViewStateMetadata) Reset()         { *m = ViewStateMetadata{} }
+func (m *ViewStateMetadata) String() string { return proto.CompactTextString(m) }
+func (*ViewStateMetadata) ProtoMessage()    {}
+
+func (m *ViewStateMetadata) GetVersion() []uint32 {
+	if m != nil {
+		return m.Version
+	}
+	return nil
+}
+
+func (m *ViewStateMetadata) GetDocumentVersionUuid() string {
+	if m != nil && m.DocumentVersionUuid != nil {
+		return *m.DocumentVersionUuid
+	}
+	return ""
+}
+
+func (m *ViewStateMetadata) GetComponent() *ComponentInfo {
+	if m != nil {
+		return m.Component
+	}
+	return nil
+}
+
+func init() {
+	proto.RegisterEnum("TSP.FieldInfo_Type", FieldInfo_Type_name, FieldInfo_Type_value)
+	proto.RegisterEnum("TSP.FieldInfo_Rule", FieldInfo_Rule_name, FieldInfo_Rule_value)
+}
--- a/vendor/code.sajari.com/docconv/iWork/TSPDatabaseMessages.pb.go
+++ b/vendor/code.sajari.com/docconv/iWork/TSPDatabaseMessages.pb.go
@@ -0,0 +1,150 @@
+// Code generated by protoc-gen-go.
+// source: TSPDatabaseMessages.proto
+// DO NOT EDIT!
+
+package TSP
+
+import proto "github.com/golang/protobuf/proto"
+import math "math"
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = math.Inf
+
+type DatabaseImageDataArchive_ImageType int32
+
+const (
+	DatabaseImageDataArchive_unknown DatabaseImageDataArchive_ImageType = 0
+	DatabaseImageDataArchive_bitmap  DatabaseImageDataArchive_ImageType = 1
+	DatabaseImageDataArchive_pdf     DatabaseImageDataArchive_ImageType = 2
+)
+
+var DatabaseImageDataArchive_ImageType_name = map[int32]string{
+	0: "unknown",
+	1: "bitmap",
+	2: "pdf",
+}
+var DatabaseImageDataArchive_ImageType_value = map[string]int32{
+	"unknown": 0,
+	"bitmap":  1,
+	"pdf":     2,
+}
+
+func (x DatabaseImageDataArchive_ImageType) Enum() *DatabaseImageDataArchive_ImageType {
+	p := new(DatabaseImageDataArchive_ImageType)
+	*p = x
+	return p
+}
+func (x DatabaseImageDataArchive_ImageType) String() string {
+	return proto.EnumName(DatabaseImageDataArchive_ImageType_name, int32(x))
+}
+func (x *DatabaseImageDataArchive_ImageType) UnmarshalJSON(data []byte) error {
+	value, err := proto.UnmarshalJSONEnum(DatabaseImageDataArchive_ImageType_value, data, "DatabaseImageDataArchive_ImageType")
+	if err != nil {
+		return err
+	}
+	*x = DatabaseImageDataArchive_ImageType(value)
+	return nil
+}
+
+type DatabaseData struct {
+	Data             *DataReference `protobuf:"bytes,1,req,name=data" json:"data,omitempty"`
+	XXX_unrecognized []byte         `json:"-"`
+}
+
+func (m *DatabaseData) Reset()         { *m = DatabaseData{} }
+func (m *DatabaseData) String() string { return proto.CompactTextString(m) }
+func (*DatabaseData) ProtoMessage()    {}
+
+func (m *DatabaseData) GetData() *DataReference {
+	if m != nil {
+		return m.Data
+	}
+	return nil
+}
+
+type DatabaseDataArchive struct {
+	Data             *Reference `protobuf:"bytes,1,opt,name=data" json:"data,omitempty"`
+	AppRelativePath  *string    `protobuf:"bytes,2,opt,name=app_relative_path" json:"app_relative_path,omitempty"`
+	DisplayName      *string    `protobuf:"bytes,3,req,name=display_name" json:"display_name,omitempty"`
+	Length           *uint64    `protobuf:"varint,4,opt,name=length" json:"length,omitempty"`
+	Hash             *uint32    `protobuf:"varint,5,opt,name=hash" json:"hash,omitempty"`
+	Sharable         *bool      `protobuf:"varint,6,req,name=sharable,def=1" json:"sharable,omitempty"`
+	XXX_unrecognized []byte     `json:"-"`
+}
+
+func (m *DatabaseDataArchive) Reset()         { *m = DatabaseDataArchive{} }
+func (m *DatabaseDataArchive) String() string { return proto.CompactTextString(m) }
+func (*DatabaseDataArchive) ProtoMessage()    {}
+
+const Default_DatabaseDataArchive_Sharable bool = true
+
+func (m *DatabaseDataArchive) GetData() *Reference {
+	if m != nil {
+		return m.Data
+	}
+	return nil
+}
+
+func (m *DatabaseDataArchive) GetAppRelativePath() string {
+	if m != nil && m.AppRelativePath != nil {
+		return *m.AppRelativePath
+	}
+	return ""
+}
+
+func (m *DatabaseDataArchive) GetDisplayName() string {
+	if m != nil && m.DisplayName != nil {
+		return *m.DisplayName
+	}
+	return ""
+}
+
+func (m *DatabaseDataArchive) GetLength() uint64 {
+	if m != nil && m.Length != nil {
+		return *m.Length
+	}
+	return 0
+}
+
+func (m *DatabaseDataArchive) GetHash() uint32 {
+	if m != nil && m.Hash != nil {
+		return *m.Hash
+	}
+	return 0
+}
+
+func (m *DatabaseDataArchive) GetSharable() bool {
+	if m != nil && m.Sharable != nil {
+		return *m.Sharable
+	}
+	return Default_DatabaseDataArchive_Sharable
+}
+
+type DatabaseImageDataArchive struct {
+	Super            *DatabaseDataArchive                `protobuf:"bytes,1,req,name=super" json:"super,omitempty"`
+	Type             *DatabaseImageDataArchive_ImageType `protobuf:"varint,2,req,name=type,enum=TSP.DatabaseImageDataArchive_ImageType" json:"type,omitempty"`
+	XXX_unrecognized []byte                              `json:"-"`
+}
+
+func (m *DatabaseImageDataArchive) Reset()         { *m = DatabaseImageDataArchive{} }
+func (m *DatabaseImageDataArchive) String() string { return proto.CompactTextString(m) }
+func (*DatabaseImageDataArchive) ProtoMessage()    {}
+
+func (m *DatabaseImageDataArchive) GetSuper() *DatabaseDataArchive {
+	if m != nil {
+		return m.Super
+	}
+	return nil
+}
+
+func (m *DatabaseImageDataArchive) GetType() DatabaseImageDataArchive_ImageType {
+	if m != nil && m.Type != nil {
+		return *m.Type
+	}
+	return DatabaseImageDataArchive_unknown
+}
+
+func init() {
+	proto.RegisterEnum("TSP.DatabaseImageDataArchive_ImageType", DatabaseImageDataArchive_ImageType_name, DatabaseImageDataArchive_ImageType_value)
+}
--- a/vendor/code.sajari.com/docconv/iWork/TSPMessages.pb.go
+++ b/vendor/code.sajari.com/docconv/iWork/TSPMessages.pb.go
@@ -0,0 +1,524 @@
+// Code generated by protoc-gen-go.
+// source: TSPMessages.proto
+// DO NOT EDIT!
+
+package TSP
+
+import proto "github.com/golang/protobuf/proto"
+import math "math"
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = math.Inf
+
+type Color_ColorModel int32
+
+const (
+	Color_rgb   Color_ColorModel = 1
+	Color_cmyk  Color_ColorModel = 2
+	Color_white Color_ColorModel = 3
+)
+
+var Color_ColorModel_name = map[int32]string{
+	1: "rgb",
+	2: "cmyk",
+	3: "white",
+}
+var Color_ColorModel_value = map[string]int32{
+	"rgb":   1,
+	"cmyk":  2,
+	"white": 3,
+}
+
+func (x Color_ColorModel) Enum() *Color_ColorModel {
+	p := new(Color_ColorModel)
+	*p = x
+	return p
+}
+func (x Color_ColorModel) String() string {
+	return proto.EnumName(Color_ColorModel_name, int32(x))
+}
+func (x *Color_ColorModel) UnmarshalJSON(data []byte) error {
+	value, err := proto.UnmarshalJSONEnum(Color_ColorModel_value, data, "Color_ColorModel")
+	if err != nil {
+		return err
+	}
+	*x = Color_ColorModel(value)
+	return nil
+}
+
+type Path_ElementType int32
+
+const (
+	Path_moveTo       Path_ElementType = 1
+	Path_lineTo       Path_ElementType = 2
+	Path_quadCurveTo  Path_ElementType = 3
+	Path_curveTo      Path_ElementType = 4
+	Path_closeSubpath Path_ElementType = 5
+)
+
+var Path_ElementType_name = map[int32]string{
+	1: "moveTo",
+	2: "lineTo",
+	3: "quadCurveTo",
+	4: "curveTo",
+	5: "closeSubpath",
+}
+var Path_ElementType_value = map[string]int32{
+	"moveTo":       1,
+	"lineTo":       2,
+	"quadCurveTo":  3,
+	"curveTo":      4,
+	"closeSubpath": 5,
+}
+
+func (x Path_ElementType) Enum() *Path_ElementType {
+	p := new(Path_ElementType)
+	*p = x
+	return p
+}
+func (x Path_ElementType) String() string {
+	return proto.EnumName(Path_ElementType_name, int32(x))
+}
+func (x *Path_ElementType) UnmarshalJSON(data []byte) error {
+	value, err := proto.UnmarshalJSONEnum(Path_ElementType_value, data, "Path_ElementType")
+	if err != nil {
+		return err
+	}
+	*x = Path_ElementType(value)
+	return nil
+}
+
+type Reference struct {
+	Identifier           *uint64 `protobuf:"varint,1,req,name=identifier" json:"identifier,omitempty"`
+	DeprecatedType       *int32  `protobuf:"varint,2,opt,name=deprecated_type" json:"deprecated_type,omitempty"`
+	DeprecatedIsExternal *bool   `protobuf:"varint,3,opt,name=deprecated_is_external" json:"deprecated_is_external,omitempty"`
+	XXX_unrecognized     []byte  `json:"-"`
+}
+
+func (m *Reference) Reset()         { *m = Reference{} }
+func (m *Reference) String() string { return proto.CompactTextString(m) }
+func (*Reference) ProtoMessage()    {}
+
+func (m *Reference) GetIdentifier() uint64 {
+	if m != nil && m.Identifier != nil {
+		return *m.Identifier
+	}
+	return 0
+}
+
+func (m *Reference) GetDeprecatedType() int32 {
+	if m != nil && m.DeprecatedType != nil {
+		return *m.DeprecatedType
+	}
+	return 0
+}
+
+func (m *Reference) GetDeprecatedIsExternal() bool {
+	if m != nil && m.DeprecatedIsExternal != nil {
+		return *m.DeprecatedIsExternal
+	}
+	return false
+}
+
+type DataReference struct {
+	Identifier       *uint64 `protobuf:"varint,1,req,name=identifier" json:"identifier,omitempty"`
+	XXX_unrecognized []byte  `json:"-"`
+}
+
+func (m *DataReference) Reset()         { *m = DataReference{} }
+func (m *DataReference) String() string { return proto.CompactTextString(m) }
+func (*DataReference) ProtoMessage()    {}
+
+func (m *DataReference) GetIdentifier() uint64 {
+	if m != nil && m.Identifier != nil {
+		return *m.Identifier
+	}
+	return 0
+}
+
+type Point struct {
+	X                *float32 `protobuf:"fixed32,1,req,name=x" json:"x,omitempty"`
+	Y                *float32 `protobuf:"fixed32,2,req,name=y" json:"y,omitempty"`
+	XXX_unrecognized []byte   `json:"-"`
+}
+
+func (m *Point) Reset()         { *m = Point{} }
+func (m *Point) String() string { return proto.CompactTextString(m) }
+func (*Point) ProtoMessage()    {}
+
+func (m *Point) GetX() float32 {
+	if m != nil && m.X != nil {
+		return *m.X
+	}
+	return 0
+}
+
+func (m *Point) GetY() float32 {
+	if m != nil && m.Y != nil {
+		return *m.Y
+	}
+	return 0
+}
+
+type Size struct {
+	Width            *float32 `protobuf:"fixed32,1,req,name=width" json:"width,omitempty"`
+	Height           *float32 `protobuf:"fixed32,2,req,name=height" json:"height,omitempty"`
+	XXX_unrecognized []byte   `json:"-"`
+}
+
+func (m *Size) Reset()         { *m = Size{} }
+func (m *Size) String() string { return proto.CompactTextString(m) }
+func (*Size) ProtoMessage()    {}
+
+func (m *Size) GetWidth() float32 {
+	if m != nil && m.Width != nil {
+		return *m.Width
+	}
+	return 0
+}
+
+func (m *Size) GetHeight() float32 {
+	if m != nil && m.Height != nil {
+		return *m.Height
+	}
+	return 0
+}
+
+type Range struct {
+	Location         *uint32 `protobuf:"varint,1,req,name=location" json:"location,omitempty"`
+	Length           *uint32 `protobuf:"varint,2,req,name=length" json:"length,omitempty"`
+	XXX_unrecognized []byte  `json:"-"`
+}
+
+func (m *Range) Reset()         { *m = Range{} }
+func (m *Range) String() string { return proto.CompactTextString(m) }
+func (*Range) ProtoMessage()    {}
+
+func (m *Range) GetLocation() uint32 {
+	if m != nil && m.Location != nil {
+		return *m.Location
+	}
+	return 0
+}
+
+func (m *Range) GetLength() uint32 {
+	if m != nil && m.Length != nil {
+		return *m.Length
+	}
+	return 0
+}
+
+type Date struct {
+	Seconds          *float64 `protobuf:"fixed64,1,req,name=seconds" json:"seconds,omitempty"`
+	XXX_unrecognized []byte   `json:"-"`
+}
+
+func (m *Date) Reset()         { *m = Date{} }
+func (m *Date) String() string { return proto.CompactTextString(m) }
+func (*Date) ProtoMessage()    {}
+
+func (m *Date) GetSeconds() float64 {
+	if m != nil && m.Seconds != nil {
+		return *m.Seconds
+	}
+	return 0
+}
+
+type IndexSet struct {
+	Ranges           []*Range `protobuf:"bytes,1,rep,name=ranges" json:"ranges,omitempty"`
+	XXX_unrecognized []byte   `json:"-"`
+}
+
+func (m *IndexSet) Reset()         { *m = IndexSet{} }
+func (m *IndexSet) String() string { return proto.CompactTextString(m) }
+func (*IndexSet) ProtoMessage()    {}
+
+func (m *IndexSet) GetRanges() []*Range {
+	if m != nil {
+		return m.Ranges
+	}
+	return nil
+}
+
+type Color struct {
+	Model            *Color_ColorModel `protobuf:"varint,1,req,name=model,enum=TSP.Color_ColorModel" json:"model,omitempty"`
+	R                *float32          `protobuf:"fixed32,3,opt,name=r" json:"r,omitempty"`
+	G                *float32          `protobuf:"fixed32,4,opt,name=g" json:"g,omitempty"`
+	B                *float32          `protobuf:"fixed32,5,opt,name=b" json:"b,omitempty"`
+	A                *float32          `protobuf:"fixed32,6,opt,name=a,def=1" json:"a,omitempty"`
+	C                *float32          `protobuf:"fixed32,7,opt,name=c" json:"c,omitempty"`
+	M                *float32          `protobuf:"fixed32,8,opt,name=m" json:"m,omitempty"`
+	Y                *float32          `protobuf:"fixed32,9,opt,name=y" json:"y,omitempty"`
+	K                *float32          `protobuf:"fixed32,10,opt,name=k" json:"k,omitempty"`
+	W                *float32          `protobuf:"fixed32,11,opt,name=w" json:"w,omitempty"`
+	XXX_unrecognized []byte            `json:"-"`
+}
+
+func (m *Color) Reset()         { *m = Color{} }
+func (m *Color) String() string { return proto.CompactTextString(m) }
+func (*Color) ProtoMessage()    {}
+
+const Default_Color_A float32 = 1
+
+func (m *Color) GetModel() Color_ColorModel {
+	if m != nil && m.Model != nil {
+		return *m.Model
+	}
+	return Color_rgb
+}
+
+func (m *Color) GetR() float32 {
+	if m != nil && m.R != nil {
+		return *m.R
+	}
+	return 0
+}
+
+func (m *Color) GetG() float32 {
+	if m != nil && m.G != nil {
+		return *m.G
+	}
+	return 0
+}
+
+func (m *Color) GetB() float32 {
+	if m != nil && m.B != nil {
+		return *m.B
+	}
+	return 0
+}
+
+func (m *Color) GetA() float32 {
+	if m != nil && m.A != nil {
+		return *m.A
+	}
+	return Default_Color_A
+}
+
+func (m *Color) GetC() float32 {
+	if m != nil && m.C != nil {
+		return *m.C
+	}
+	return 0
+}
+
+func (m *Color) GetM() float32 {
+	if m != nil && m.M != nil {
+		return *m.M
+	}
+	return 0
+}
+
+func (m *Color) GetY() float32 {
+	if m != nil && m.Y != nil {
+		return *m.Y
+	}
+	return 0
+}
+
+func (m *Color) GetK() float32 {
+	if m != nil && m.K != nil {
+		return *m.K
+	}
+	return 0
+}
+
+func (m *Color) GetW() float32 {
+	if m != nil && m.W != nil {
+		return *m.W
+	}
+	return 0
+}
+
+type Path struct {
+	Elements         []*Path_Element `protobuf:"bytes,1,rep,name=elements" json:"elements,omitempty"`
+	XXX_unrecognized []byte          `json:"-"`
+}
+
+func (m *Path) Reset()         { *m = Path{} }
+func (m *Path) String() string { return proto.CompactTextString(m) }
+func (*Path) ProtoMessage()    {}
+
+func (m *Path) GetElements() []*Path_Element {
+	if m != nil {
+		return m.Elements
+	}
+	return nil
+}
+
+type Path_Element struct {
+	Type             *Path_ElementType `protobuf:"varint,1,req,name=type,enum=TSP.Path_ElementType" json:"type,omitempty"`
+	Points           []*Point          `protobuf:"bytes,2,rep,name=points" json:"points,omitempty"`
+	XXX_unrecognized []byte            `json:"-"`
+}
+
+func (m *Path_Element) Reset()         { *m = Path_Element{} }
+func (m *Path_Element) String() string { return proto.CompactTextString(m) }
+func (*Path_Element) ProtoMessage()    {}
+
+func (m *Path_Element) GetType() Path_ElementType {
+	if m != nil && m.Type != nil {
+		return *m.Type
+	}
+	return Path_moveTo
+}
+
+func (m *Path_Element) GetPoints() []*Point {
+	if m != nil {
+		return m.Points
+	}
+	return nil
+}
+
+type ReferenceDictionary struct {
+	Entries          []*ReferenceDictionary_Entry `protobuf:"bytes,1,rep,name=entries" json:"entries,omitempty"`
+	XXX_unrecognized []byte                       `json:"-"`
+}
+
+func (m *ReferenceDictionary) Reset()         { *m = ReferenceDictionary{} }
+func (m *ReferenceDictionary) String() string { return proto.CompactTextString(m) }
+func (*ReferenceDictionary) ProtoMessage()    {}
+
+func (m *ReferenceDictionary) GetEntries() []*ReferenceDictionary_Entry {
+	if m != nil {
+		return m.Entries
+	}
+	return nil
+}
+
+type ReferenceDictionary_Entry struct {
+	Key              *Reference `protobuf:"bytes,1,req,name=key" json:"key,omitempty"`
+	Value            *Reference `protobuf:"bytes,2,req,name=value" json:"value,omitempty"`
+	XXX_unrecognized []byte     `json:"-"`
+}
+
+func (m *ReferenceDictionary_Entry) Reset()         { *m = ReferenceDictionary_Entry{} }
+func (m *ReferenceDictionary_Entry) String() string { return proto.CompactTextString(m) }
+func (*ReferenceDictionary_Entry) ProtoMessage()    {}
+
+func (m *ReferenceDictionary_Entry) GetKey() *Reference {
+	if m != nil {
+		return m.Key
+	}
+	return nil
+}
+
+func (m *ReferenceDictionary_Entry) GetValue() *Reference {
+	if m != nil {
+		return m.Value
+	}
+	return nil
+}
+
+type PasteboardObject struct {
+	Stylesheet       *Reference   `protobuf:"bytes,1,opt,name=stylesheet" json:"stylesheet,omitempty"`
+	Drawables        []*Reference `protobuf:"bytes,2,rep,name=drawables" json:"drawables,omitempty"`
+	Styles           []*Reference `protobuf:"bytes,3,rep,name=styles" json:"styles,omitempty"`
+	Theme            *Reference   `protobuf:"bytes,4,opt,name=theme" json:"theme,omitempty"`
+	WpStorage        *Reference   `protobuf:"bytes,5,opt,name=wp_storage" json:"wp_storage,omitempty"`
+	GuideStorage     *Reference   `protobuf:"bytes,9,opt,name=guide_storage" json:"guide_storage,omitempty"`
+	AppNativeObject  *Reference   `protobuf:"bytes,6,opt,name=app_native_object" json:"app_native_object,omitempty"`
+	IsTextPrimary    *bool        `protobuf:"varint,7,opt,name=is_text_primary,def=0" json:"is_text_primary,omitempty"`
+	IsSmart          *bool        `protobuf:"varint,8,opt,name=is_smart,def=0" json:"is_smart,omitempty"`
+	XXX_unrecognized []byte       `json:"-"`
+}
+
+func (m *PasteboardObject) Reset()         { *m = PasteboardObject{} }
+func (m *PasteboardObject) String() string { return proto.CompactTextString(m) }
+func (*PasteboardObject) ProtoMessage()    {}
+
+const Default_PasteboardObject_IsTextPrimary bool = false
+const Default_PasteboardObject_IsSmart bool = false
+
+func (m *PasteboardObject) GetStylesheet() *Reference {
+	if m != nil {
+		return m.Stylesheet
+	}
+	return nil
+}
+
+func (m *PasteboardObject) GetDrawables() []*Reference {
+	if m != nil {
+		return m.Drawables
+	}
+	return nil
+}
+
+func (m *PasteboardObject) GetStyles() []*Reference {
+	if m != nil {
+		return m.Styles
+	}
+	return nil
+}
+
+func (m *PasteboardObject) GetTheme() *Reference {
+	if m != nil {
+		return m.Theme
+	}
+	return nil
+}
+
+func (m *PasteboardObject) GetWpStorage() *Reference {
+	if m != nil {
+		return m.WpStorage
+	}
+	return nil
+}
+
+func (m *PasteboardObject) GetGuideStorage() *Reference {
+	if m != nil {
+		return m.GuideStorage
+	}
+	return nil
+}
+
+func (m *PasteboardObject) GetAppNativeObject() *Reference {
+	if m != nil {
+		return m.AppNativeObject
+	}
+	return nil
+}
+
+func (m *PasteboardObject) GetIsTextPrimary() bool {
+	if m != nil && m.IsTextPrimary != nil {
+		return *m.IsTextPrimary
+	}
+	return Default_PasteboardObject_IsTextPrimary
+}
+
+func (m *PasteboardObject) GetIsSmart() bool {
+	if m != nil && m.IsSmart != nil {
+		return *m.IsSmart
+	}
+	return Default_PasteboardObject_IsSmart
+}
+
+type ObjectContainer struct {
+	Identifier       *uint32      `protobuf:"varint,1,opt,name=identifier" json:"identifier,omitempty"`
+	Objects          []*Reference `protobuf:"bytes,2,rep,name=objects" json:"objects,omitempty"`
+	XXX_unrecognized []byte       `json:"-"`
+}
+
+func (m *ObjectContainer) Reset()         { *m = ObjectContainer{} }
+func (m *ObjectContainer) String() string { return proto.CompactTextString(m) }
+func (*ObjectContainer) ProtoMessage()    {}
+
+func (m *ObjectContainer) GetIdentifier() uint32 {
+	if m != nil && m.Identifier != nil {
+		return *m.Identifier
+	}
+	return 0
+}
+
+func (m *ObjectContainer) GetObjects() []*Reference {
+	if m != nil {
+		return m.Objects
+	}
+	return nil
+}
+
+func init() {
+	proto.RegisterEnum("TSP.Color_ColorModel", Color_ColorModel_name, Color_ColorModel_value)
+	proto.RegisterEnum("TSP.Path_ElementType", Path_ElementType_name, Path_ElementType_value)
+}
--- a/vendor/code.sajari.com/docconv/image.go
+++ b/vendor/code.sajari.com/docconv/image.go
@@ -0,0 +1,17 @@
+// +build !ocr
+
+package docconv
+
+import (
+	"fmt"
+	"io"
+)
+
+// ConvertImage converts images to text.
+// Requires gosseract (ocr build tag).
+func ConvertImage(r io.Reader) (string, map[string]string, error) {
+	return "", nil, fmt.Errorf("docconv not built with `ocr` build tag")
+}
+
+// SetImageLanguages sets the languages parameter passed to gosseract.
+func SetImageLanguages(...string) {}
--- a/vendor/code.sajari.com/docconv/image_ocr.go
+++ b/vendor/code.sajari.com/docconv/image_ocr.go
@@ -0,0 +1,51 @@
+// +build ocr
+
+package docconv
+
+import (
+	"fmt"
+	"io"
+	"sync"
+
+	"github.com/otiai10/gosseract/v2"
+)
+
+var config = struct {
+	langs []string
+	sync.Mutex
+}{
+	langs: []string{"eng"},
+}
+
+func SetImageLanguages(l ...string) {
+	config.Lock()
+	config.langs = l
+	config.Unlock()
+}
+
+// ConvertImage converts images to text.
+// Requires gosseract.
+func ConvertImage(r io.Reader) (string, map[string]string, error) {
+	f, err := NewLocalFile(r)
+	if err != nil {
+		return "", nil, fmt.Errorf("error creating local file: %v", err)
+	}
+	defer f.Done()
+
+	meta := make(map[string]string)
+
+	client := gosseract.NewClient()
+	defer client.Close()
+
+	config.Lock()
+	defer config.Unlock()
+
+	client.SetLanguage(config.langs...)
+	client.SetImage(f.Name())
+	text, err := client.Text()
+	if err != nil {
+		return "", nil, err
+	}
+
+	return text, meta, nil
+}
--- a/vendor/code.sajari.com/docconv/local.go
+++ b/vendor/code.sajari.com/docconv/local.go
@@ -0,0 +1,51 @@
+package docconv
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+)
+
+// LocalFile is a type which wraps an *os.File.  See NewLocalFile for more details.
+type LocalFile struct {
+	*os.File
+
+	unlink bool
+}
+
+// NewLocalFile ensures that there is a file which contains the data provided by r.  If r is
+// actually an instance of *os.File then this file is used, otherwise a temporary file is
+// created and the data from r copied into it.  Callers must call Done() when
+// the LocalFile is no longer needed to ensure all resources are cleaned up.
+func NewLocalFile(r io.Reader) (*LocalFile, error) {
+	if f, ok := r.(*os.File); ok {
+		return &LocalFile{
+			File: f,
+		}, nil
+	}
+
+	f, err := ioutil.TempFile(os.TempDir(), "/docconv")
+	if err != nil {
+		return nil, fmt.Errorf("error creating temporary file: %v", err)
+	}
+	_, err = io.Copy(f, r)
+	if err != nil {
+		f.Close()
+		os.Remove(f.Name())
+		return nil, fmt.Errorf("error copying data into temporary file: %v", err)
+	}
+
+	return &LocalFile{
+		File:   f,
+		unlink: true,
+	}, nil
+}
+
+// Done cleans up all resources.
+func (l *LocalFile) Done() {
+	l.Close()
+	if l.unlink {
+		os.Remove(l.Name())
+	}
+}
--- a/vendor/code.sajari.com/docconv/odt.go
+++ b/vendor/code.sajari.com/docconv/odt.go
@@ -0,0 +1,69 @@
+package docconv
+
+import (
+	"archive/zip"
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"time"
+)
+
+// ConvertODT converts a ODT file to text
+func ConvertODT(r io.Reader) (string, map[string]string, error) {
+	meta := make(map[string]string)
+	var textBody string
+
+	b, err := ioutil.ReadAll(r)
+	if err != nil {
+		return "", nil, err
+	}
+	zr, err := zip.NewReader(bytes.NewReader(b), int64(len(b)))
+	if err != nil {
+		return "", nil, fmt.Errorf("error unzipping data: %v", err)
+	}
+
+	for _, f := range zr.File {
+		switch f.Name {
+		case "meta.xml":
+			rc, err := f.Open()
+			if err != nil {
+				return "", nil, fmt.Errorf("error extracting '%v' from archive: %v", f.Name, err)
+			}
+			defer rc.Close()
+
+			info, err := XMLToMap(rc)
+			if err != nil {
+				return "", nil, fmt.Errorf("error parsing '%v': %v", f.Name, err)
+			}
+
+			if tmp, ok := info["creator"]; ok {
+				meta["Author"] = tmp
+			}
+			if tmp, ok := info["date"]; ok {
+				if t, err := time.Parse("2006-01-02T15:04:05", tmp); err == nil {
+					meta["ModifiedDate"] = fmt.Sprintf("%d", t.Unix())
+				}
+			}
+			if tmp, ok := info["creation-date"]; ok {
+				if t, err := time.Parse("2006-01-02T15:04:05", tmp); err == nil {
+					meta["CreatedDate"] = fmt.Sprintf("%d", t.Unix())
+				}
+			}
+
+		case "content.xml":
+			rc, err := f.Open()
+			if err != nil {
+				return "", nil, fmt.Errorf("error extracting '%v' from archive: %v", f.Name, err)
+			}
+			defer rc.Close()
+
+			textBody, err = XMLToText(rc, []string{"br", "p", "tab"}, []string{}, true)
+			if err != nil {
+				return "", nil, fmt.Errorf("error parsing '%v': %v", f.Name, err)
+			}
+		}
+	}
+
+	return textBody, meta, nil
+}
--- a/vendor/code.sajari.com/docconv/pages.go
+++ b/vendor/code.sajari.com/docconv/pages.go
@@ -0,0 +1,60 @@
+package docconv
+
+import (
+	"archive/zip"
+	"bufio"
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"strings"
+
+	"github.com/golang/protobuf/proto"
+
+	"code.sajari.com/docconv/iWork"
+	"code.sajari.com/docconv/snappy"
+)
+
+// ConvertPages converts a Pages file to text.
+func ConvertPages(r io.Reader) (string, map[string]string, error) {
+	meta := make(map[string]string)
+	var textBody string
+
+	b, err := ioutil.ReadAll(r)
+	if err != nil {
+		return "", nil, fmt.Errorf("error reading data: %v", err)
+	}
+
+	zr, err := zip.NewReader(bytes.NewReader(b), int64(len(b)))
+	if err != nil {
+		return "", nil, fmt.Errorf("error unzipping data: %v", err)
+	}
+
+	for _, f := range zr.File {
+		if strings.HasSuffix(f.Name, "Preview.pdf") {
+			// There is a preview PDF version we can use
+			if rc, err := f.Open(); err == nil {
+				return ConvertPDF(rc)
+			}
+		}
+		if f.Name == "index.xml" {
+			// There's an XML version we can use
+			if rc, err := f.Open(); err == nil {
+				return ConvertXML(rc)
+			}
+		}
+		if f.Name == "Index/Document.iwa" {
+			rc, _ := f.Open()
+			defer rc.Close()
+			bReader := bufio.NewReader(snappy.NewReader(io.MultiReader(strings.NewReader("\xff\x06\x00\x00sNaPpY"), rc)))
+			archiveLength, err := binary.ReadVarint(bReader)
+			archiveInfoData, err := ioutil.ReadAll(io.LimitReader(bReader, archiveLength))
+			archiveInfo := &TSP.ArchiveInfo{}
+			err = proto.Unmarshal(archiveInfoData, archiveInfo)
+			fmt.Println("archiveInfo:", archiveInfo, err)
+		}
+	}
+
+	return textBody, meta, nil
+}
--- a/vendor/code.sajari.com/docconv/pdf.go
+++ b/vendor/code.sajari.com/docconv/pdf.go
@@ -0,0 +1,30 @@
+// +build !ocr
+
+package docconv
+
+import (
+	"fmt"
+	"io"
+)
+
+func ConvertPDF(r io.Reader) (string, map[string]string, error) {
+
+	f, err := NewLocalFile(r)
+	if err != nil {
+		return "", nil, fmt.Errorf("error creating local file: %v", err)
+	}
+	defer f.Done()
+
+	bodyResult, metaResult, convertErr := ConvertPDFText(f.Name())
+	if convertErr != nil {
+		return "", nil, convertErr
+	}
+	if bodyResult.err != nil {
+		return "", nil, bodyResult.err
+	}
+	if metaResult.err != nil {
+		return "", nil, metaResult.err
+	}
+	return bodyResult.body, metaResult.meta, nil
+
+}
--- a/vendor/code.sajari.com/docconv/pdf_ocr.go
+++ b/vendor/code.sajari.com/docconv/pdf_ocr.go
@@ -0,0 +1,161 @@
+// +build ocr
+
+package docconv
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync"
+)
+
+var (
+	exts = []string{".jpg", ".tif", ".tiff", ".png", ".pbm"}
+)
+
+func compareExt(ext string, exts []string) bool {
+	for _, e := range exts {
+		if ext == e {
+			return true
+		}
+	}
+	return false
+}
+
+func cleanupTemp(tmpDir string) {
+	err := os.RemoveAll(tmpDir)
+	if err != nil {
+		log.Println(err)
+	}
+}
+
+func ConvertPDFImages(path string) (BodyResult, error) {
+	bodyResult := BodyResult{}
+
+	tmp, err := ioutil.TempDir(os.TempDir(), "tmp-imgs-")
+	if err != nil {
+		bodyResult.err = err
+		return bodyResult, err
+	}
+	tmpDir := fmt.Sprintf("%s/", tmp)
+
+	defer cleanupTemp(tmpDir)
+
+	_, err = exec.Command("pdfimages", "-j", path, tmpDir).Output()
+	if err != nil {
+		return bodyResult, err
+	}
+
+	filePaths := []string{}
+
+	walkFunc := func(path string, info os.FileInfo, err error) error {
+		path, err = filepath.Abs(path)
+		if err != nil {
+			return err
+		}
+
+		if compareExt(filepath.Ext(path), exts) {
+			filePaths = append(filePaths, path)
+		}
+		return nil
+	}
+	filepath.Walk(tmpDir, walkFunc)
+
+	fileLength := len(filePaths)
+
+	if fileLength < 1 {
+		return bodyResult, nil
+	}
+
+	var wg sync.WaitGroup
+
+	data := make(chan string, fileLength)
+
+	wg.Add(fileLength)
+
+	for _, p := range filePaths {
+		go func(pathFile string) {
+			defer wg.Done()
+			f, err := os.Open(pathFile)
+			if err != nil {
+				return
+			}
+
+			defer f.Close()
+			out, _, err := ConvertImage(f)
+			if err != nil {
+				return
+			}
+
+			data <- out
+
+		}(p)
+	}
+
+	wg.Wait()
+
+	close(data)
+
+	for str := range data {
+		bodyResult.body += str + " "
+	}
+
+	return bodyResult, nil
+}
+
+// PdfHasImage verify if `path` (PDF) has images
+func PDFHasImage(path string) bool {
+	cmd := "pdffonts -l 5 %s | tail -n +3 | cut -d' ' -f1 | sort | uniq"
+	out, err := exec.Command("bash", "-c", fmt.Sprintf(cmd, path)).Output()
+	if err != nil {
+		log.Println(err)
+		return false
+	}
+	if string(out) == "" {
+		return true
+	}
+	return false
+}
+
+func ConvertPDF(r io.Reader) (string, map[string]string, error) {
+	f, err := NewLocalFile(r)
+	if err != nil {
+		return "", nil, fmt.Errorf("error creating local file: %v", err)
+	}
+	defer f.Done()
+
+	bodyResult, metaResult, textConvertErr := ConvertPDFText(f.Name())
+	if textConvertErr != nil {
+		return "", nil, textConvertErr
+	}
+	if bodyResult.err != nil {
+		return "", nil, bodyResult.err
+	}
+	if metaResult.err != nil {
+		return "", nil, metaResult.err
+	}
+
+	if !PDFHasImage(f.Name()) {
+		return bodyResult.body, metaResult.meta, nil
+	}
+
+	imageConvertResult, imageConvertErr := ConvertPDFImages(f.Name())
+	if imageConvertErr != nil {
+		log.Println(imageConvertErr)
+		return bodyResult.body, metaResult.meta, nil
+	}
+	if imageConvertResult.err != nil {
+		log.Println(imageConvertResult.err)
+		return bodyResult.body, metaResult.meta, nil
+	}
+
+	fullBody := strings.Join([]string{bodyResult.body, imageConvertResult.body}, " ")
+
+	return fullBody, metaResult.meta, nil
+
+}
--- a/vendor/code.sajari.com/docconv/pdf_text.go
+++ b/vendor/code.sajari.com/docconv/pdf_text.go
@@ -0,0 +1,84 @@
+package docconv
+
+import (
+	"fmt"
+	"os/exec"
+	"strings"
+	"time"
+)
+
+// Meta data
+type MetaResult struct {
+	meta map[string]string
+	err  error
+}
+
+type BodyResult struct {
+	body string
+	err  error
+}
+
+// Convert PDF
+
+func ConvertPDFText(path string) (BodyResult, MetaResult, error) {
+	metaResult := MetaResult{meta: make(map[string]string)}
+	bodyResult := BodyResult{}
+	mr := make(chan MetaResult, 1)
+	go func() {
+		metaStr, err := exec.Command("pdfinfo", path).Output()
+		if err != nil {
+			metaResult.err = err
+			mr <- metaResult
+			return
+		}
+
+		// Parse meta output
+		for _, line := range strings.Split(string(metaStr), "\n") {
+			if parts := strings.SplitN(line, ":", 2); len(parts) > 1 {
+				metaResult.meta[strings.TrimSpace(parts[0])] = strings.TrimSpace(parts[1])
+			}
+		}
+
+		// Convert parsed meta
+		if x, ok := metaResult.meta["ModDate"]; ok {
+			if t, ok := pdfTimeLayouts.Parse(x); ok {
+				metaResult.meta["ModifiedDate"] = fmt.Sprintf("%d", t.Unix())
+			}
+		}
+		if x, ok := metaResult.meta["CreationDate"]; ok {
+			if t, ok := pdfTimeLayouts.Parse(x); ok {
+				metaResult.meta["CreatedDate"] = fmt.Sprintf("%d", t.Unix())
+			}
+		}
+
+		mr <- metaResult
+	}()
+
+	br := make(chan BodyResult, 1)
+	go func() {
+		body, err := exec.Command("pdftotext", "-q", "-nopgbrk", "-enc", "UTF-8", "-eol", "unix", path, "-").Output()
+		if err != nil {
+			bodyResult.err = err
+		}
+
+		bodyResult.body = string(body)
+
+		br <- bodyResult
+	}()
+
+	return <-br, <-mr, nil
+}
+
+var pdfTimeLayouts = timeLayouts{time.ANSIC, "Mon Jan _2 15:04:05 2006 MST"}
+
+type timeLayouts []string
+
+func (tl timeLayouts) Parse(x string) (time.Time, bool) {
+	for _, layout := range tl {
+		t, err := time.Parse(layout, x)
+		if err == nil {
+			return t, true
+		}
+	}
+	return time.Time{}, false
+}
--- a/vendor/code.sajari.com/docconv/pptx.go
+++ b/vendor/code.sajari.com/docconv/pptx.go
@@ -0,0 +1,67 @@
+package docconv
+
+import (
+	"archive/zip"
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"strings"
+)
+
+// ConvertPptx converts an MS PowerPoint pptx file to text.
+func ConvertPptx(r io.Reader) (string, map[string]string, error) {
+	var size int64
+
+	// Common case: if the reader is a file (or trivial wrapper), avoid
+	// loading it all into memory.
+	var ra io.ReaderAt
+	if f, ok := r.(interface {
+		io.ReaderAt
+		Stat() (os.FileInfo, error)
+	}); ok {
+		si, err := f.Stat()
+		if err != nil {
+			return "", nil, err
+		}
+		size = si.Size()
+		ra = f
+	} else {
+		b, err := ioutil.ReadAll(r)
+		if err != nil {
+			return "", nil, nil
+		}
+		size = int64(len(b))
+		ra = bytes.NewReader(b)
+	}
+
+	zr, err := zip.NewReader(ra, size)
+	if err != nil {
+		return "", nil, fmt.Errorf("could not unzip: %v", err)
+	}
+
+	zipFiles := mapZipFiles(zr.File)
+
+	contentTypeDefinition, err := getContentTypeDefinition(zipFiles["[Content_Types].xml"])
+	if err != nil {
+		return "", nil, err
+	}
+
+	meta := make(map[string]string)
+	var textBody string
+	for _, override := range contentTypeDefinition.Overrides {
+		f := zipFiles[override.PartName]
+
+		switch override.ContentType {
+		case "application/vnd.openxmlformats-officedocument.presentationml.slide+xml",
+			"application/vnd.openxmlformats-officedocument.drawingml.diagramData+xml":
+			body, err := parseDocxText(f)
+			if err != nil {
+				return "", nil, fmt.Errorf("could not parse pptx: %v", err)
+			}
+			textBody += body + "\n"
+		}
+	}
+	return strings.TrimSuffix(textBody, "\n"), meta, nil
+}
--- a/vendor/code.sajari.com/docconv/rtf.go
+++ b/vendor/code.sajari.com/docconv/rtf.go
@@ -0,0 +1,52 @@
+package docconv
+
+import (
+	"fmt"
+	"io"
+	"os/exec"
+	"strings"
+	"time"
+)
+
+// ConvertRTF converts RTF files to text.
+func ConvertRTF(r io.Reader) (string, map[string]string, error) {
+	f, err := NewLocalFile(r)
+	if err != nil {
+		return "", nil, fmt.Errorf("error creating local file: %v", err)
+	}
+	defer f.Done()
+
+	var output string
+	tmpOutput, err := exec.Command("unrtf", "--nopict", "--text", f.Name()).Output()
+	if err != nil {
+		return "", nil, fmt.Errorf("unrtf error: %v", err)
+	}
+
+	// Step through content looking for meta data and stripping out comments
+	meta := make(map[string]string)
+	for _, line := range strings.Split(string(tmpOutput), "\n") {
+		if parts := strings.SplitN(line, ":", 2); len(parts) > 1 {
+			meta[strings.TrimSpace(parts[0])] = strings.TrimSpace(parts[1])
+		}
+		if len(line) > 4 && line[:4] != "### " {
+			output += line + "\n"
+		}
+	}
+
+	// Identify meta data
+	if tmp, ok := meta["AUTHOR"]; ok {
+		meta["Author"] = tmp
+	}
+	if tmp, ok := meta["### creation date"]; ok {
+		if t, err := time.Parse("02 January 2006 15:04", tmp); err == nil {
+			meta["CreatedDate"] = fmt.Sprintf("%d", t.Unix())
+		}
+	}
+	if tmp, ok := meta["### revision date"]; ok {
+		if t, err := time.Parse("02 January 2006 15:04", tmp); err == nil {
+			meta["ModifiedDate"] = fmt.Sprintf("%d", t.Unix())
+		}
+	}
+
+	return output, meta, nil
+}
--- a/vendor/code.sajari.com/docconv/snappy/LICENSE
+++ b/vendor/code.sajari.com/docconv/snappy/LICENSE
@@ -0,0 +1,27 @@
+Copyright (c) 2011 The Snappy-Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
--- a/vendor/code.sajari.com/docconv/snappy/README
+++ b/vendor/code.sajari.com/docconv/snappy/README
@@ -0,0 +1,13 @@
+This is a Snappy library for the Go programming language that has been modified to work with Apple files, which fail to set CRC checks and stream identifiers. This version is a total hack, so if you want to use snappy for other projects **DO NOT USE THIS VERSION**. Use the proper version as per below:
+
+To download and install from source:
+$ go get code.google.com/p/snappy-go/snappy
+
+
+
+Unless otherwise noted, the Snappy-Go source files are distributed
+under the BSD-style license found in the LICENSE file.
+
+Contributions should follow the same procedure as for the Go project:
+http://golang.org/doc/contribute.html
+
--- a/vendor/code.sajari.com/docconv/snappy/decode.go
+++ b/vendor/code.sajari.com/docconv/snappy/decode.go
@@ -0,0 +1,297 @@
+// Copyright 2011 The Snappy-Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package snappy
+
+import (
+	"encoding/binary"
+	"errors"
+	"io"
+)
+
+var (
+	// ErrCorrupt reports that the input is invalid.
+	ErrCorrupt = errors.New("snappy: corrupt input")
+	// ErrUnsupported reports that the input isn't supported.
+	ErrUnsupported = errors.New("snappy: unsupported input")
+)
+
+// DecodedLen returns the length of the decoded block.
+func DecodedLen(src []byte) (int, error) {
+	v, _, err := decodedLen(src)
+	return v, err
+}
+
+// decodedLen returns the length of the decoded block and the number of bytes
+// that the length header occupied.
+func decodedLen(src []byte) (blockLen, headerLen int, err error) {
+	v, n := binary.Uvarint(src)
+	if n == 0 {
+		return 0, 0, ErrCorrupt
+	}
+	if uint64(int(v)) != v {
+		return 0, 0, errors.New("snappy: decoded block is too large")
+	}
+	return int(v), n, nil
+}
+
+// Decode returns the decoded form of src. The returned slice may be a sub-
+// slice of dst if dst was large enough to hold the entire decoded block.
+// Otherwise, a newly allocated slice will be returned.
+// It is valid to pass a nil dst.
+func Decode(dst, src []byte) ([]byte, error) {
+	dLen, s, err := decodedLen(src)
+	if err != nil {
+		return nil, err
+	}
+	if len(dst) < dLen {
+		dst = make([]byte, dLen)
+	}
+
+	var d, offset, length int
+	for s < len(src) {
+		switch src[s] & 0x03 {
+		case tagLiteral:
+			x := uint(src[s] >> 2)
+			switch {
+			case x < 60:
+				s += 1
+			case x == 60:
+				s += 2
+				if s > len(src) {
+					return nil, ErrCorrupt
+				}
+				x = uint(src[s-1])
+			case x == 61:
+				s += 3
+				if s > len(src) {
+					return nil, ErrCorrupt
+				}
+				x = uint(src[s-2]) | uint(src[s-1])<<8
+			case x == 62:
+				s += 4
+				if s > len(src) {
+					return nil, ErrCorrupt
+				}
+				x = uint(src[s-3]) | uint(src[s-2])<<8 | uint(src[s-1])<<16
+			case x == 63:
+				s += 5
+				if s > len(src) {
+					return nil, ErrCorrupt
+				}
+				x = uint(src[s-4]) | uint(src[s-3])<<8 | uint(src[s-2])<<16 | uint(src[s-1])<<24
+			}
+			length = int(x + 1)
+			if length <= 0 {
+				return nil, errors.New("snappy: unsupported literal length")
+			}
+			if length > len(dst)-d || length > len(src)-s {
+				return nil, ErrCorrupt
+			}
+			copy(dst[d:], src[s:s+length])
+			d += length
+			s += length
+			continue
+
+		case tagCopy1:
+			s += 2
+			if s > len(src) {
+				return nil, ErrCorrupt
+			}
+			length = 4 + int(src[s-2])>>2&0x7
+			offset = int(src[s-2])&0xe0<<3 | int(src[s-1])
+
+		case tagCopy2:
+			s += 3
+			if s > len(src) {
+				return nil, ErrCorrupt
+			}
+			length = 1 + int(src[s-3])>>2
+			offset = int(src[s-2]) | int(src[s-1])<<8
+
+		case tagCopy4:
+			return nil, errors.New("snappy: unsupported COPY_4 tag")
+		}
+
+		end := d + length
+		if offset > d || end > len(dst) {
+			return nil, ErrCorrupt
+		}
+		for ; d < end; d++ {
+			dst[d] = dst[d-offset]
+		}
+	}
+	if d != dLen {
+		return nil, ErrCorrupt
+	}
+	return dst[:d], nil
+}
+
+// NewReader returns a new Reader that decompresses from r, using the framing
+// format described at
+// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt
+func NewReader(r io.Reader) *Reader {
+	return &Reader{
+		r:       r,
+		decoded: make([]byte, maxUncompressedChunkLen),
+		buf:     make([]byte, MaxEncodedLen(maxUncompressedChunkLen)+checksumSize),
+	}
+}
+
+// Reader is an io.Reader than can read Snappy-compressed bytes.
+type Reader struct {
+	r       io.Reader
+	err     error
+	decoded []byte
+	buf     []byte
+	// decoded[i:j] contains decoded bytes that have not yet been passed on.
+	i, j       int
+	readHeader bool
+}
+
+// Reset discards any buffered data, resets all state, and switches the Snappy
+// reader to read from r. This permits reusing a Reader rather than allocating
+// a new one.
+func (r *Reader) Reset(reader io.Reader) {
+	r.r = reader
+	r.err = nil
+	r.i = 0
+	r.j = 0
+	r.readHeader = false
+}
+
+func (r *Reader) readFull(p []byte) (ok bool) {
+	if _, r.err = io.ReadFull(r.r, p); r.err != nil {
+		if r.err == io.ErrUnexpectedEOF {
+			r.err = ErrCorrupt
+		}
+		return false
+	}
+	return true
+}
+
+// Read satisfies the io.Reader interface.
+func (r *Reader) Read(p []byte) (int, error) {
+	if r.err != nil {
+		return 0, r.err
+	}
+	for {
+		if r.i < r.j {
+			n := copy(p, r.decoded[r.i:r.j])
+			r.i += n
+			return n, nil
+		}
+		if !r.readFull(r.buf[:4]) {
+			return 0, r.err
+		}
+		chunkType := r.buf[0]
+		if !r.readHeader {
+			if chunkType != chunkTypeStreamIdentifier {
+				r.err = ErrCorrupt
+				return 0, r.err
+			}
+			r.readHeader = true
+		}
+		chunkLen := int(r.buf[1]) | int(r.buf[2])<<8 | int(r.buf[3])<<16
+		if chunkLen > len(r.buf) {
+			r.err = ErrUnsupported
+			return 0, r.err
+		}
+
+		// The chunk types are specified at
+		// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt
+		switch chunkType {
+		case chunkTypeCompressedData:
+			// Section 4.2. Compressed data (chunk type 0x00).
+			/*
+				if chunkLen < checksumSize {
+					r.err = ErrCorrupt
+					return 0, r.err
+				}
+			*/
+			buf := r.buf[:chunkLen]
+			if !r.readFull(buf) {
+				return 0, r.err
+			}
+			/*
+				checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24
+				buf = buf[checksumSize:]
+			*/
+			n, err := DecodedLen(buf)
+			if err != nil {
+				r.err = err
+				return 0, r.err
+			}
+			if n > len(r.decoded) {
+				r.err = ErrCorrupt
+				return 0, r.err
+			}
+			if _, err := Decode(r.decoded, buf); err != nil {
+				r.err = err
+				return 0, r.err
+			}
+			/*
+				if crc(r.decoded[:n]) != checksum {
+					r.err = ErrCorrupt
+					return 0, r.err
+				}
+			*/
+			r.i, r.j = 0, n
+			continue
+
+		case chunkTypeUncompressedData:
+			// Section 4.3. Uncompressed data (chunk type 0x01).
+			if chunkLen < checksumSize {
+				r.err = ErrCorrupt
+				return 0, r.err
+			}
+			buf := r.buf[:checksumSize]
+			if !r.readFull(buf) {
+				return 0, r.err
+			}
+			checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24
+			// Read directly into r.decoded instead of via r.buf.
+			n := chunkLen - checksumSize
+			if !r.readFull(r.decoded[:n]) {
+				return 0, r.err
+			}
+			if crc(r.decoded[:n]) != checksum {
+				r.err = ErrCorrupt
+				return 0, r.err
+			}
+			r.i, r.j = 0, n
+			continue
+
+		case chunkTypeStreamIdentifier:
+			// Section 4.1. Stream identifier (chunk type 0xff).
+			if chunkLen != len(magicBody) {
+				r.err = ErrCorrupt
+				return 0, r.err
+			}
+			if !r.readFull(r.buf[:len(magicBody)]) {
+				return 0, r.err
+			}
+			for i := 0; i < len(magicBody); i++ {
+				if r.buf[i] != magicBody[i] {
+					r.err = ErrCorrupt
+					return 0, r.err
+				}
+			}
+			continue
+		}
+
+		if chunkType <= 0x7f {
+			// Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f).
+			r.err = ErrUnsupported
+			return 0, r.err
+
+		} else {
+			// Section 4.4 Padding (chunk type 0xfe).
+			// Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd).
+			if !r.readFull(r.buf[:chunkLen]) {
+				return 0, r.err
+			}
+		}
+	}
+}
--- a/vendor/code.sajari.com/docconv/snappy/encode.go
+++ b/vendor/code.sajari.com/docconv/snappy/encode.go
@@ -0,0 +1,258 @@
+// Copyright 2011 The Snappy-Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package snappy
+
+import (
+	"encoding/binary"
+	"io"
+)
+
+// We limit how far copy back-references can go, the same as the C++ code.
+const maxOffset = 1 << 15
+
+// emitLiteral writes a literal chunk and returns the number of bytes written.
+func emitLiteral(dst, lit []byte) int {
+	i, n := 0, uint(len(lit)-1)
+	switch {
+	case n < 60:
+		dst[0] = uint8(n)<<2 | tagLiteral
+		i = 1
+	case n < 1<<8:
+		dst[0] = 60<<2 | tagLiteral
+		dst[1] = uint8(n)
+		i = 2
+	case n < 1<<16:
+		dst[0] = 61<<2 | tagLiteral
+		dst[1] = uint8(n)
+		dst[2] = uint8(n >> 8)
+		i = 3
+	case n < 1<<24:
+		dst[0] = 62<<2 | tagLiteral
+		dst[1] = uint8(n)
+		dst[2] = uint8(n >> 8)
+		dst[3] = uint8(n >> 16)
+		i = 4
+	case int64(n) < 1<<32:
+		dst[0] = 63<<2 | tagLiteral
+		dst[1] = uint8(n)
+		dst[2] = uint8(n >> 8)
+		dst[3] = uint8(n >> 16)
+		dst[4] = uint8(n >> 24)
+		i = 5
+	default:
+		panic("snappy: source buffer is too long")
+	}
+	if copy(dst[i:], lit) != len(lit) {
+		panic("snappy: destination buffer is too short")
+	}
+	return i + len(lit)
+}
+
+// emitCopy writes a copy chunk and returns the number of bytes written.
+func emitCopy(dst []byte, offset, length int) int {
+	i := 0
+	for length > 0 {
+		x := length - 4
+		if 0 <= x && x < 1<<3 && offset < 1<<11 {
+			dst[i+0] = uint8(offset>>8)&0x07<<5 | uint8(x)<<2 | tagCopy1
+			dst[i+1] = uint8(offset)
+			i += 2
+			break
+		}
+
+		x = length
+		if x > 1<<6 {
+			x = 1 << 6
+		}
+		dst[i+0] = uint8(x-1)<<2 | tagCopy2
+		dst[i+1] = uint8(offset)
+		dst[i+2] = uint8(offset >> 8)
+		i += 3
+		length -= x
+	}
+	return i
+}
+
+// Encode returns the encoded form of src. The returned slice may be a sub-
+// slice of dst if dst was large enough to hold the entire encoded block.
+// Otherwise, a newly allocated slice will be returned.
+// It is valid to pass a nil dst.
+func Encode(dst, src []byte) ([]byte, error) {
+	if n := MaxEncodedLen(len(src)); len(dst) < n {
+		dst = make([]byte, n)
+	}
+
+	// The block starts with the varint-encoded length of the decompressed bytes.
+	d := binary.PutUvarint(dst, uint64(len(src)))
+
+	// Return early if src is short.
+	if len(src) <= 4 {
+		if len(src) != 0 {
+			d += emitLiteral(dst[d:], src)
+		}
+		return dst[:d], nil
+	}
+
+	// Initialize the hash table. Its size ranges from 1<<8 to 1<<14 inclusive.
+	const maxTableSize = 1 << 14
+	shift, tableSize := uint(32-8), 1<<8
+	for tableSize < maxTableSize && tableSize < len(src) {
+		shift--
+		tableSize *= 2
+	}
+	var table [maxTableSize]int
+
+	// Iterate over the source bytes.
+	var (
+		s   int // The iterator position.
+		t   int // The last position with the same hash as s.
+		lit int // The start position of any pending literal bytes.
+	)
+	for s+3 < len(src) {
+		// Update the hash table.
+		b0, b1, b2, b3 := src[s], src[s+1], src[s+2], src[s+3]
+		h := uint32(b0) | uint32(b1)<<8 | uint32(b2)<<16 | uint32(b3)<<24
+		p := &table[(h*0x1e35a7bd)>>shift]
+		// We need to to store values in [-1, inf) in table. To save
+		// some initialization time, (re)use the table's zero value
+		// and shift the values against this zero: add 1 on writes,
+		// subtract 1 on reads.
+		t, *p = *p-1, s+1
+		// If t is invalid or src[s:s+4] differs from src[t:t+4], accumulate a literal byte.
+		if t < 0 || s-t >= maxOffset || b0 != src[t] || b1 != src[t+1] || b2 != src[t+2] || b3 != src[t+3] {
+			s++
+			continue
+		}
+		// Otherwise, we have a match. First, emit any pending literal bytes.
+		if lit != s {
+			d += emitLiteral(dst[d:], src[lit:s])
+		}
+		// Extend the match to be as long as possible.
+		s0 := s
+		s, t = s+4, t+4
+		for s < len(src) && src[s] == src[t] {
+			s++
+			t++
+		}
+		// Emit the copied bytes.
+		d += emitCopy(dst[d:], s-t, s-s0)
+		lit = s
+	}
+
+	// Emit any final pending literal bytes and return.
+	if lit != len(src) {
+		d += emitLiteral(dst[d:], src[lit:])
+	}
+	return dst[:d], nil
+}
+
+// MaxEncodedLen returns the maximum length of a snappy block, given its
+// uncompressed length.
+func MaxEncodedLen(srcLen int) int {
+	// Compressed data can be defined as:
+	//    compressed := item* literal*
+	//    item       := literal* copy
+	//
+	// The trailing literal sequence has a space blowup of at most 62/60
+	// since a literal of length 60 needs one tag byte + one extra byte
+	// for length information.
+	//
+	// Item blowup is trickier to measure. Suppose the "copy" op copies
+	// 4 bytes of data. Because of a special check in the encoding code,
+	// we produce a 4-byte copy only if the offset is < 65536. Therefore
+	// the copy op takes 3 bytes to encode, and this type of item leads
+	// to at most the 62/60 blowup for representing literals.
+	//
+	// Suppose the "copy" op copies 5 bytes of data. If the offset is big
+	// enough, it will take 5 bytes to encode the copy op. Therefore the
+	// worst case here is a one-byte literal followed by a five-byte copy.
+	// That is, 6 bytes of input turn into 7 bytes of "compressed" data.
+	//
+	// This last factor dominates the blowup, so the final estimate is:
+	return 32 + srcLen + srcLen/6
+}
+
+// NewWriter returns a new Writer that compresses to w, using the framing
+// format described at
+// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt
+func NewWriter(w io.Writer) *Writer {
+	return &Writer{
+		w:   w,
+		enc: make([]byte, MaxEncodedLen(maxUncompressedChunkLen)),
+	}
+}
+
+// Writer is an io.Writer than can write Snappy-compressed bytes.
+type Writer struct {
+	w           io.Writer
+	err         error
+	enc         []byte
+	buf         [checksumSize + chunkHeaderSize]byte
+	wroteHeader bool
+}
+
+// Reset discards the writer's state and switches the Snappy writer to write to
+// w. This permits reusing a Writer rather than allocating a new one.
+func (w *Writer) Reset(writer io.Writer) {
+	w.w = writer
+	w.err = nil
+	w.wroteHeader = false
+}
+
+// Write satisfies the io.Writer interface.
+func (w *Writer) Write(p []byte) (n int, errRet error) {
+	if w.err != nil {
+		return 0, w.err
+	}
+	if !w.wroteHeader {
+		copy(w.enc, magicChunk)
+		if _, err := w.w.Write(w.enc[:len(magicChunk)]); err != nil {
+			w.err = err
+			return n, err
+		}
+		w.wroteHeader = true
+	}
+	for len(p) > 0 {
+		var uncompressed []byte
+		if len(p) > maxUncompressedChunkLen {
+			uncompressed, p = p[:maxUncompressedChunkLen], p[maxUncompressedChunkLen:]
+		} else {
+			uncompressed, p = p, nil
+		}
+		checksum := crc(uncompressed)
+
+		// Compress the buffer, discarding the result if the improvement
+		// isn't at least 12.5%.
+		chunkType := uint8(chunkTypeCompressedData)
+		chunkBody, err := Encode(w.enc, uncompressed)
+		if err != nil {
+			w.err = err
+			return n, err
+		}
+		if len(chunkBody) >= len(uncompressed)-len(uncompressed)/8 {
+			chunkType, chunkBody = chunkTypeUncompressedData, uncompressed
+		}
+
+		chunkLen := 4 + len(chunkBody)
+		w.buf[0] = chunkType
+		w.buf[1] = uint8(chunkLen >> 0)
+		w.buf[2] = uint8(chunkLen >> 8)
+		w.buf[3] = uint8(chunkLen >> 16)
+		w.buf[4] = uint8(checksum >> 0)
+		w.buf[5] = uint8(checksum >> 8)
+		w.buf[6] = uint8(checksum >> 16)
+		w.buf[7] = uint8(checksum >> 24)
+		if _, err = w.w.Write(w.buf[:]); err != nil {
+			w.err = err
+			return n, err
+		}
+		if _, err = w.w.Write(chunkBody); err != nil {
+			w.err = err
+			return n, err
+		}
+		n += len(uncompressed)
+	}
+	return n, nil
+}
--- a/vendor/code.sajari.com/docconv/snappy/snappy.go
+++ b/vendor/code.sajari.com/docconv/snappy/snappy.go
@@ -0,0 +1,68 @@
+// Copyright 2011 The Snappy-Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package snappy implements the snappy block-based compression format.
+// It aims for very high speeds and reasonable compression.
+//
+// The C++ snappy implementation is at http://code.google.com/p/snappy/
+package snappy
+
+import (
+	"hash/crc32"
+)
+
+/*
+Each encoded block begins with the varint-encoded length of the decoded data,
+followed by a sequence of chunks. Chunks begin and end on byte boundaries. The
+first byte of each chunk is broken into its 2 least and 6 most significant bits
+called l and m: l ranges in [0, 4) and m ranges in [0, 64). l is the chunk tag.
+Zero means a literal tag. All other values mean a copy tag.
+
+For literal tags:
+  - If m < 60, the next 1 + m bytes are literal bytes.
+  - Otherwise, let n be the little-endian unsigned integer denoted by the next
+    m - 59 bytes. The next 1 + n bytes after that are literal bytes.
+
+For copy tags, length bytes are copied from offset bytes ago, in the style of
+Lempel-Ziv compression algorithms. In particular:
+  - For l == 1, the offset ranges in [0, 1<<11) and the length in [4, 12).
+    The length is 4 + the low 3 bits of m. The high 3 bits of m form bits 8-10
+    of the offset. The next byte is bits 0-7 of the offset.
+  - For l == 2, the offset ranges in [0, 1<<16) and the length in [1, 65).
+    The length is 1 + m. The offset is the little-endian unsigned integer
+    denoted by the next 2 bytes.
+  - For l == 3, this tag is a legacy format that is no longer supported.
+*/
+const (
+	tagLiteral = 0x00
+	tagCopy1   = 0x01
+	tagCopy2   = 0x02
+	tagCopy4   = 0x03
+)
+
+const (
+	checksumSize    = 4
+	chunkHeaderSize = 4
+	magicChunk      = "\xff\x06\x00\x00" + magicBody
+	magicBody       = "sNaPpY"
+	// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt says
+	// that "the uncompressed data in a chunk must be no longer than 65536 bytes".
+	maxUncompressedChunkLen = 65536
+)
+
+const (
+	chunkTypeCompressedData   = 0x00
+	chunkTypeUncompressedData = 0x01
+	chunkTypePadding          = 0xfe
+	chunkTypeStreamIdentifier = 0xff
+)
+
+var crcTable = crc32.MakeTable(crc32.Castagnoli)
+
+// crc implements the checksum specified in section 3 of
+// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt
+func crc(b []byte) uint32 {
+	c := crc32.Update(0, crcTable, b)
+	return uint32(c>>15|c<<17) + 0xa282ead8
+}
--- a/vendor/code.sajari.com/docconv/tidy.go
+++ b/vendor/code.sajari.com/docconv/tidy.go
@@ -0,0 +1,32 @@
+package docconv
+
+import (
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+)
+
+// Tidy attempts to tidy up XML.
+// Errors & warnings are deliberately suppressed as underlying tools
+// throw warnings very easily.
+func Tidy(r io.Reader, xmlIn bool) ([]byte, error) {
+	f, err := ioutil.TempFile(os.TempDir(), "/docconv")
+	if err != nil {
+		return nil, err
+	}
+	defer os.Remove(f.Name())
+	io.Copy(f, r)
+
+	var output []byte
+	if xmlIn {
+		output, err = exec.Command("tidy", "-xml", "-numeric", "-asxml", "-quiet", "-utf8", f.Name()).Output()
+	} else {
+		output, err = exec.Command("tidy", "-numeric", "-asxml", "-quiet", "-utf8", f.Name()).Output()
+	}
+
+	if err != nil && err.Error() != "exit status 1" {
+		return nil, err
+	}
+	return output, nil
+}
--- a/vendor/code.sajari.com/docconv/url.go
+++ b/vendor/code.sajari.com/docconv/url.go
@@ -0,0 +1,31 @@
+package docconv
+
+import (
+	"bytes"
+	"io"
+
+	"github.com/advancedlogic/GoOse"
+)
+
+// ConvertURL fetches the HTML page at the URL given in the io.Reader.
+func ConvertURL(input io.Reader, readability bool) (string, map[string]string, error) {
+	meta := make(map[string]string)
+
+	buf := new(bytes.Buffer)
+	_, err := buf.ReadFrom(input)
+	if err != nil {
+		return "", nil, err
+	}
+
+	g := goose.New()
+	article, err := g.ExtractFromURL(buf.String())
+	if err != nil {
+		return "", nil, err
+	}
+
+	meta["title"] = article.Title
+	meta["description"] = article.MetaDescription
+	meta["image"] = article.TopImage
+
+	return article.CleanedText, meta, nil
+}
--- a/vendor/code.sajari.com/docconv/xml.go
+++ b/vendor/code.sajari.com/docconv/xml.go
@@ -0,0 +1,98 @@
+package docconv
+
+import (
+	"bytes"
+	"encoding/xml"
+	"fmt"
+	"io"
+)
+
+// ConvertXML converts an XML file to text.
+func ConvertXML(r io.Reader) (string, map[string]string, error) {
+	meta := make(map[string]string)
+	cleanXML, err := Tidy(r, true)
+	if err != nil {
+		return "", nil, fmt.Errorf("tidy error: %v", err)
+	}
+	result, err := XMLToText(bytes.NewReader(cleanXML), []string{}, []string{}, true)
+	if err != nil {
+		return "", nil, fmt.Errorf("error from XMLToText: %v", err)
+	}
+	return result, meta, nil
+}
+
+// XMLToText converts XML to plain text given how to treat elements.
+func XMLToText(r io.Reader, breaks []string, skip []string, strict bool) (string, error) {
+	var result string
+
+	dec := xml.NewDecoder(r)
+	dec.Strict = strict
+	for {
+		t, err := dec.Token()
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return "", err
+		}
+
+		switch v := t.(type) {
+		case xml.CharData:
+			result += string(v)
+		case xml.StartElement:
+			for _, breakElement := range breaks {
+				if v.Name.Local == breakElement {
+					result += "\n"
+				}
+			}
+			for _, skipElement := range skip {
+				if v.Name.Local == skipElement {
+					depth := 1
+					for {
+						t, err := dec.Token()
+						if err != nil {
+							// An io.EOF here is actually an error.
+							return "", err
+						}
+
+						switch t.(type) {
+						case xml.StartElement:
+							depth++
+						case xml.EndElement:
+							depth--
+						}
+
+						if depth == 0 {
+							break
+						}
+					}
+				}
+			}
+		}
+	}
+	return result, nil
+}
+
+// XMLToMap converts XML to a nested string map.
+func XMLToMap(r io.Reader) (map[string]string, error) {
+	m := make(map[string]string)
+	dec := xml.NewDecoder(r)
+	var tagName string
+	for {
+		t, err := dec.Token()
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return nil, err
+		}
+
+		switch v := t.(type) {
+		case xml.StartElement:
+			tagName = string(v.Name.Local)
+		case xml.CharData:
+			m[tagName] = string(v)
+		}
+	}
+	return m, nil
+}
--- a/vendor/github.com/JalfResi/justext/.gitignore
+++ b/vendor/github.com/JalfResi/justext/.gitignore
@@ -0,0 +1,32 @@
+
+src/_go_.6
+
+src/_obj/ourscienceistight/gojustext.a
+
+src/example/_go_.6
+
+src/example/gojustext
+
+src/main.go_old
+
+src/.DS_Store
+
+src/stoplists/.DS_Store
+
+src/example/*.html
+
+src/example/*.htm
+
+src/stoplists/*.txt
+
+src/example/hp-envy-14-spectre-review
+
+src/example/t.txt
+
+src/example/t2.txt
+
+gojustext
+
+gojustext.sublime-project
+
+gojustext.sublime-workspace
--- a/vendor/github.com/JalfResi/justext/README.md
+++ b/vendor/github.com/JalfResi/justext/README.md
@@ -0,0 +1,48 @@
+justext
+=======
+
+A Go package that implements the JusText boilerplate removal algorithm (http://code.google.com/p/justext/)
+
+## Install
+
+    go get github.com/JalfResi/justext
+
+And import:
+
+    import "github.com/JalfResi/justext"
+
+## Usage
+
+Supports all stoplist files available at http://code.google.com/p/justext/source/browse/#svn%2Ftrunk%2Fjustext%2Fstoplists
+
+Justext expects valid HTML; it is your responsability to ensure that valid HTML is passed to Justext. To make things easier 
+I have written a CGO wrapper around libtidy which you can find here: [github.com/JalfResi/GoTidy](https://github.com/JalfResi/GoTidy)
+In the future, once exp/html is part of the standard packages I will refactor JusText to accept only valid HTML documents/strings.
+
+Justext use the reader-writer idiom, alowing you to setup the reader with a common configuration and just pump out 
+articles to the writer.
+
+Example usage:
+
+    // Create a justext reader from another reader
+    reader := justext.NewReader(os.Stdin)
+    
+    // Configure the reader
+    reader.LengthLow = 70
+    reader.LengthHigh = 200
+    reader.Stoplist = stoplist // The stoplist map[string]bool
+    reader.StopwordsLow = 0.3
+    reader.StopwordsHigh = 0.32
+    reader.MaxLinkDensity = 0.2
+    reader.MaxHeadingDistance = 200
+    reader.NoHeadings = false
+    
+    // Read from the reader to generate a paragraph set
+    paragraphSet, _ := reader.ReadAll()
+    
+    // Create a writer from another writer
+    writer := justext.NewWriter(os.Stdout)
+    // Write the paragraph set to the writer
+    writer.WriteAll(paragraphSet)
+    
+    
--- a/vendor/github.com/JalfResi/justext/TODO.txt
+++ b/vendor/github.com/JalfResi/justext/TODO.txt
@@ -0,0 +1,3 @@
+TODO
+====
+- Stoplists need to be separtae subpackages (use the init() package method of registration)
--- a/vendor/github.com/JalfResi/justext/classifyParagraphs.go
+++ b/vendor/github.com/JalfResi/justext/classifyParagraphs.go
@@ -0,0 +1,66 @@
+package justext
+
+import (
+	"regexp"
+	"strings"
+)
+
+var findHeadings *regexp.Regexp = regexp.MustCompile("(^h[123456]|.h[123456])")
+var copyrightChar *regexp.Regexp = regexp.MustCompile("(\u0161|&copy)")
+var findSelect *regexp.Regexp = regexp.MustCompile("(^select|.select)")
+
+func classifyParagraphs(paragraphs []*Paragraph, stoplist map[string]bool, lengthLow int, lengthHigh int, stopwordsLow float64, stopwordsHigh float64, maxLinkDensity float64, noHeadings bool) {
+	for _, paragraph := range paragraphs {
+		var length int = len(paragraph.Text)
+		var stopwordCount int = 0
+		for _, word := range strings.Split(paragraph.Text, " ") {
+			if _, ok := stoplist[word]; ok {
+				stopwordCount += 1
+			}
+		}
+
+		var stopwordDensity float64 = 0.0
+		var linkDensity float64 = 0.0
+		var wordCount int = paragraph.WordCount
+
+		if wordCount > 0 {
+			stopwordDensity = 1.0 * float64(stopwordCount) / float64(wordCount)
+			linkDensity = float64(paragraph.LinkedCharCount) / float64(length)
+		}
+
+		paragraph.StopwordCount = stopwordCount
+		paragraph.StopwordDensity = stopwordDensity
+		paragraph.LinkDensity = linkDensity
+		paragraph.Heading = bool(!noHeadings && findHeadings.MatchString(paragraph.DomPath))
+
+		if linkDensity > maxLinkDensity {
+			paragraph.CfClass = "bad"
+		} else if copyrightChar.MatchString(paragraph.Text) {
+			paragraph.CfClass = "bad"
+		} else if findSelect.MatchString(paragraph.DomPath) {
+			paragraph.CfClass = "bad"
+		} else {
+			if length < lengthLow {
+				if paragraph.LinkedCharCount > 0 {
+					paragraph.CfClass = "bad"
+				} else {
+					paragraph.CfClass = "short"
+				}
+			} else {
+				if stopwordDensity >= stopwordsHigh {
+					if length > lengthHigh {
+						paragraph.CfClass = "good"
+					} else {
+						paragraph.CfClass = "neargood"
+					}
+				} else {
+					if stopwordDensity >= stopwordsLow {
+						paragraph.CfClass = "neargood"
+					} else {
+						paragraph.CfClass = "bad"
+					}
+				}
+			}
+		}
+	}
+}
--- a/vendor/github.com/JalfResi/justext/defaultTemplate.go
+++ b/vendor/github.com/JalfResi/justext/defaultTemplate.go
@@ -0,0 +1,50 @@
+package justext
+
+import (
+	"bytes"
+	"compress/gzip"
+	"io"
+	"reflect"
+	"unsafe"
+)
+
+var _DefaultTemplate = "" +
+	"\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\x74\x90\x3d\x6e\xc4\x20" +
+	"\x10\x46\x6b\xfb\x14\xc8\x4a\x8d\xfb\x08\x53\x24\x45\x92\x26\x8a" +
+	"\x14\x5f\x60\x6c\x88\x41\xc2\x80\x80\x22\x2b\xc4\xdd\x17\xc3\x7a" +
+	"\xff\xb4\x5b\x81\xde\x3c\xe6\x1b\x86\x88\xb0\x2a\xda\x92\xc9\xb0" +
+	"\x03\x6d\x63\x7c\xd1\xe6\xcd\x48\xc5\x9d\x55\x10\x38\x7a\x1d\x10" +
+	"\xfe\xbe\x26\x29\x65\xc9\x81\x5e\x38\xc2\x3f\xe0\x60\x71\x60\x85" +
+	"\xcf\xb4\x89\x51\xfe\xa1\x2f\xff\x61\x0c\x43\xf8\x5d\x81\x2f\xb4" +
+	"\x62\xfc\xc9\x81\x49\xbd\x14\xd2\x10\x41\x63\x1c\x9d\x5c\x7f\x2d" +
+	"\xcc\xb9\xcf\xc8\xff\x43\x4a\xa4\x17\xb4\xf8\x5c\x79\x7e\x12\xed" +
+	"\x63\xd1\xd2\x5c\xac\xae\x66\x35\xfb\xfc\xa8\xe4\x69\x13\xd0\xed" +
+	"\x4f\xf6\x86\x68\xde\x26\x1b\xba\xe9\x52\xea\x9e\x85\xdc\x25\x94" +
+	"\xcb\x7e\x92\xbe\x6e\x2c\x4f\xbd\x2d\xf0\x18\x00\x00\xff\xff\x2c" +
+	"\xc5\xf5\x5d\x47\x01\x00\x00"
+
+// DefaultTemplate returns the binary data for a given file.
+func DefaultTemplate() []byte {
+	// This bit of black magic ensures we do not get
+	// unneccesary memcpy's and can read directly from
+	// the .rodata section.
+	var empty [0]byte
+	sx := (*reflect.StringHeader)(unsafe.Pointer(&_DefaultTemplate))
+	b := empty[:]
+	bx := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+	bx.Data = sx.Data
+	bx.Len = len(_DefaultTemplate)
+	bx.Cap = bx.Len
+
+	gz, err := gzip.NewReader(bytes.NewBuffer(b))
+
+	if err != nil {
+		panic("Decompression failed: " + err.Error())
+	}
+
+	var buf bytes.Buffer
+	io.Copy(&buf, gz)
+	gz.Close()
+
+	return buf.Bytes()
+}
--- a/vendor/github.com/JalfResi/justext/detailedTemplate.go
+++ b/vendor/github.com/JalfResi/justext/detailedTemplate.go
@@ -0,0 +1,181 @@
+package justext
+
+import (
+	"bytes"
+	"compress/gzip"
+	"io"
+	"reflect"
+	"unsafe"
+)
+
+var _DetailedTemplate = "" +
+	"\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\xe4\x59\x59\x73\xe2\x3a" +
+	"\xf6\x7f\x4e\x3e\x85\xff\xfc\x6f\x4d\x27\x45\x9a\x1d\x02\x09\x64" +
+	"\xc6\x18\x13\x76\x02\x06\x02\x99\x9a\xea\x12\xb6\x6c\x14\xaf\xb1" +
+	"\x65\x8c\xc9\xf0\xdd\x47\x32\x66\x0b\x24\x4d\xcf\x43\xdf\x5b\x35" +
+	"\xbc\xd8\x3e\x3a\xfb\xf9\x49\x3a\x12\xc5\x19\xd6\xb5\x87\xcb\xa2" +
+	"\x83\x7d\x0d\x3e\x5c\x5e\xfc\x03\xe9\x96\x69\x63\xc6\xb5\xb5\xab" +
+	"\x19\xc6\xd6\x5d\x3c\x2e\x9b\x06\x76\x62\x8a\x69\x2a\x1a\x04\x16" +
+	"\x72\x62\xa2\xa9\xc7\x45\xc7\xf9\xbb\x0c\x74\xa4\xf9\xa5\xae\x05" +
+	"\x8d\xa8\x00\x0c\xe7\x2e\x93\x48\xdc\xa4\x13\x89\xbf\x39\xee\xd4" +
+	"\x81\xb8\xa4\x01\x8c\x8c\x1b\xd1\xb7\x91\xa6\x21\xf1\x3b\x5c\xe0" +
+	"\xed\xc7\x8d\x62\x43\xa8\x06\xa4\x39\x82\xd8\x00\x3a\x74\xe0\x4d" +
+	"\x20\x40\x89\xd7\xf7\x97\x17\x97\x17\x53\x53\xf2\x2f\x2f\xde\x2f" +
+	"\x2f\x2e\xa8\x0b\xdf\xd7\xe6\xee\x98\x6f\xd4\x20\x43\x0d\x7e\xbb" +
+	"\x61\x1c\xf2\xf8\xee\x40\x1b\xc9\xf7\x1b\x3e\x0f\x22\x65\x86\xef" +
+	"\x18\xe2\xc9\x3d\x43\x88\x3a\xb0\x15\xa2\x56\x83\x32\x21\x02\x17" +
+	"\x9b\xf7\x3b\xa2\xbd\x66\xdd\x50\x3d\x24\xe1\xd9\x1d\x93\x4f\x24" +
+	"\xac\x05\xf9\x5e\x5d\x5e\x5e\x38\x16\x30\x62\x0e\x36\x2d\xcf\xb4" +
+	"\x25\x86\x3a\x83\x89\x83\xdf\x25\x28\x9a\x36\x71\xd7\x34\xee\x18" +
+	"\xd7\x90\xa0\xad\x21\x03\x86\x22\x16\x4d\x96\x74\xc3\x58\xb1\x19" +
+	"\x04\x12\x32\x14\xfa\x3a\x05\x81\x34\x43\x7e\xa1\x95\x6c\x76\x6d" +
+	"\x85\x92\x02\x9d\x40\x43\x0a\x51\xf7\xea\x3a\x18\xc9\x7e\x38\x12" +
+	"\x3a\x4a\x1c\xb8\x63\x52\x5b\xfe\x90\x3a\x35\x31\x36\xf5\xfd\x01" +
+	"\x0b\x48\xd4\x62\x18\x6d\xfa\x88\x1e\x06\x9c\xde\xc6\x77\xe4\xec" +
+	"\xc6\xcb\x29\x10\x55\xc5\x36\x49\x70\xdf\x45\x53\x33\xed\x3b\xe6" +
+	"\xff\x45\x59\xdc\x4a\xed\x71\x1f\xa6\x3d\x13\xa6\x3d\x20\x3a\x68" +
+	"\x09\xef\x98\x64\x2c\x0d\xf5\xad\xe4\x5e\x26\x4e\xd8\x90\x25\x29" +
+	"\xe4\x94\xd0\x3c\x66\x01\x1b\x28\x36\xb0\x66\x3f\x24\x88\x01\xd2" +
+	"\x9c\x0f\x39\x4c\x65\x76\x39\x94\x35\x13\x10\xfb\x41\x84\x21\x49" +
+	"\x42\x8e\xa5\x01\x82\x19\xc3\xdc\x56\xe7\xb4\x5a\x0c\xa6\x1a\xdc" +
+	"\xba\x45\x4a\x0d\x6d\xea\x92\x06\x2c\x87\x04\xb0\x79\x3b\x43\x05" +
+	"\xde\x06\x77\x58\x8a\xe3\x12\x85\xa5\xd8\x0d\xac\xcd\x92\x74\x59" +
+	"\x0b\xc6\x31\x35\x24\x91\x6c\xc8\xf2\x57\x36\x83\xda\xad\x0d\xdf" +
+	"\x7c\xc6\xb2\x29\xd4\x89\x08\x0f\x4c\x15\xe4\xc2\x97\xa6\x68\xd9" +
+	"\x7e\xa6\x43\x06\xe0\x4c\x77\x19\x6c\x9f\xe9\x31\xb6\xbf\x40\xcb" +
+	"\xf9\x4e\x7f\xa9\x66\xe7\x77\x0c\xfa\xd0\x24\xeb\x4b\xb8\xee\xac" +
+	"\x21\x45\x2b\x78\xff\x71\x25\xf9\x9e\x24\x40\xdf\x11\x83\x09\xba" +
+	"\x9e\x56\x17\x5b\xd8\x4d\x35\x53\x54\xf7\x16\x96\x74\x72\xcd\x30" +
+	"\x0b\xe7\x4a\x2a\xb3\xfe\xde\x73\xc9\x86\x16\xa4\x36\x0d\x33\x7c" +
+	"\xfd\x30\x8e\x74\xa0\x10\x44\xd2\xf5\x59\x02\x18\xdc\x05\xdf\x71" +
+	"\xcb\x50\xee\xa7\xc0\x81\xb9\xcc\x0d\x1a\x95\xbb\x7d\x2f\xd1\x7c" +
+	"\x54\x4c\x96\xfc\x3a\xc2\x70\xc6\x0f\x15\xf2\x56\xce\xd3\xef\x3e" +
+	"\xc7\x4e\xe8\x53\xb3\x6a\x52\x83\x52\x1b\x09\x8d\xef\x8d\xfa\x99" +
+	"\xd4\x5b\x6a\x34\xee\xa5\x2b\x6d\xd6\xe7\x17\x3e\xdb\xe3\xde\xf8" +
+	"\x1e\xab\xf3\xc1\x93\x43\x7c\xaf\x8c\xc2\x77\x87\xef\xb5\x5f\x0b" +
+	"\x62\xa3\x8d\x9b\x53\x0c\x52\x42\x27\x5a\x49\xe4\x07\xc6\xb8\x76" +
+	"\x9b\x03\xcd\x54\x35\x31\x9d\x98\xcf\x03\x55\x74\xb3\xf5\x45\x4b" +
+	"\x5d\xf0\x6d\xae\x96\xb5\xa6\x3d\x76\x24\x69\x28\xa3\x2d\x13\xc3" +
+	"\x7a\xaf\xde\x86\xd9\xa7\xbe\xe8\xf3\x6d\x1f\xa4\x9f\xd2\xec\x5b" +
+	"\x53\x10\xdc\xe9\xe8\x09\x18\x8b\xaa\x66\xc1\x16\x18\xa3\x7e\x79" +
+	"\xf8\x72\xcb\xab\x13\x7b\xde\x19\xbd\xb1\xd5\x41\x6e\x22\x8d\x27" +
+	"\xed\xc9\x68\x24\x24\xeb\x8f\x2f\xc9\x49\x33\xca\xc7\x1f\xcb\xe2" +
+	"\x04\xd5\x52\xb5\xea\xf8\x35\x5b\xad\x8d\xbb\x99\x6a\x66\x91\x6b" +
+	"\x45\x93\xaf\xb7\xac\x50\x43\x9e\x33\x28\x3c\xb5\x66\xfd\xf9\x4b" +
+	"\x27\x6a\x44\x0b\x2e\x1a\x62\x81\xef\xcb\x4e\x74\xdc\x99\xb0\x9c" +
+	"\x97\xcc\x0f\xa7\xdd\x64\xe7\xcd\x86\x6a\xf2\x79\x6e\xcd\xa3\x65" +
+	"\xf1\x55\x4d\x34\x52\x95\xd6\x44\x28\x24\x6a\xe5\x6a\x05\x60\x2e" +
+	"\xaf\xf8\x8b\xac\x06\xe2\x51\x3b\x5d\x4b\xb3\x03\xbe\xda\x7d\x92" +
+	"\x2b\xad\xaa\xfb\xda\xcd\x65\xdc\xe6\xa0\xba\x30\x3b\xe9\x27\xcb" +
+	"\xc8\x38\x6f\x78\xc9\xdd\xbe\xe9\xa3\x7c\x42\x97\x04\x77\xe0\x4d" +
+	"\xbb\x59\x3d\xe5\x8e\x86\x80\xeb\x75\xc1\x70\x2c\x88\x42\x83\x2b" +
+	"\x14\x16\x8e\x5d\x90\x0a\xf5\x42\x2a\xee\x2f\x9b\x8b\x81\xee\xb9" +
+	"\x43\x0f\x3e\xaa\xb5\x78\x5a\x2d\x64\x66\x15\x29\xe5\x2f\xec\x11" +
+	"\x6c\x69\x62\xa7\xc7\x06\xa5\xe9\x0f\xb3\xbc\xad\x36\x14\x45\x29" +
+	"\x95\xae\xf7\x10\x2a\x6a\xa6\x03\xa5\xf3\x31\xfa\x57\x06\xe4\x24" +
+	"\x04\x24\xf7\xe6\x83\x32\xa5\x4e\xf9\x10\x90\x76\xea\xb9\x22\xa6" +
+	"\xf9\x3a\xeb\x3f\x2e\x3d\x5e\x68\xe8\xbc\xa0\xd6\x39\x5e\x18\xb2" +
+	"\x1c\x6f\xb1\x1e\x37\x44\x93\x19\xaf\x4f\xca\xdc\xc0\x01\x8b\x2e" +
+	"\x28\x74\x32\x6c\x34\x8e\x53\x8d\xc7\xbc\x5b\x4e\xfa\xb2\x66\xbd" +
+	"\xf8\x46\x3a\x2d\xc7\x93\xc3\x44\xed\x51\x9d\x73\x76\xaa\x30\xcd" +
+	"\xb3\xe3\x72\xbf\x10\x2d\xeb\x70\xc2\xa2\x42\x2d\x9a\xcb\x03\x9c" +
+	"\xeb\xb3\x48\xf7\xf3\xcd\xc6\x30\xdb\xcf\x9a\x23\xb9\x65\x56\xfb" +
+	"\x39\x76\x32\x07\xe5\x7c\xc3\x7a\x75\xf9\xa1\x1d\x9d\x22\x24\xa4" +
+	"\x9b\x29\xce\x8c\x3e\xce\xbd\x6a\x45\x89\xf2\x26\xa8\x44\xf9\xc4" +
+	"\xb2\xae\xbe\x2e\x15\xd9\x06\x99\xda\xeb\x74\xb1\xe4\xf8\xe9\x22" +
+	"\xa5\xd4\xb3\xea\x20\xd1\x2b\xa8\xcf\x0d\xdb\xee\x8d\x9e\x0c\x61" +
+	"\xf6\xfa\xe2\xc5\xdd\x4e\x4e\xe8\xcc\xf3\x8e\xe1\xd7\x6e\xcb\x09" +
+	"\xa9\x2b\xd8\x04\x65\xec\x52\xe8\x69\xfa\x14\x5a\x4b\x47\xf4\xa2" +
+	"\x65\xde\x29\xf4\x58\xac\x76\x9f\x47\x39\xb8\x18\xc1\xf2\x4b\x56" +
+	"\xab\x22\xfc\xc8\xdb\xb9\x96\xd9\x83\x75\xa1\x3e\x00\xa3\x51\xc3" +
+	"\x18\x71\xee\x4b\xed\x16\x4b\x2d\x6d\xda\xec\xea\x6d\x37\x3a\x98" +
+	"\xf5\x75\x1b\x8f\x5b\x5a\xd9\xd1\x0c\x33\xda\x28\x2c\x5e\x26\x96" +
+	"\xe8\x77\xd9\xbe\xef\x67\xd8\x9a\xb1\x48\x35\xe7\xb3\xee\x5b\xbd" +
+	"\x62\xa9\xac\x17\xf7\xf8\x0a\x28\x24\x97\x5c\x62\x32\x6e\xb6\x9a" +
+	"\xaa\x69\xf0\x60\xd4\x66\x5f\x8c\x4e\x54\xc9\x65\x35\x38\xcb\x3e" +
+	"\xd5\x13\x4e\x35\xa9\x3b\x8f\xd6\x78\x1e\x97\x85\x6e\xf6\x6d\x7a" +
+	"\x2b\x4d\x92\xfd\xf4\xab\xe5\x56\xd5\x4c\x6b\x68\xd8\x05\x2b\x21" +
+	"\xb6\xf2\x05\x2e\xeb\x95\x73\x95\x44\x66\x6e\xd4\xe1\x30\xd9\xd2" +
+	"\x59\xcf\x91\xd5\xb6\xe4\x55\xab\xfd\x84\x37\x1f\xa6\x33\x12\x1a" +
+	"\xa6\x9b\xf1\xd9\xf3\xb8\xde\x02\xd1\x38\x07\x86\xcb\x0c\x8e\xa3" +
+	"\x46\x5c\xa9\xbc\xc8\x5a\x5a\x2f\xe4\xc4\x71\xdb\xfb\x0c\xec\xc5" +
+	"\x78\xd8\x96\x16\x1d\xd1\x46\x16\x26\xfd\xa9\xec\x1a\x22\xed\xba" +
+	"\x18\x67\x66\x7a\x3f\x80\x34\x07\x86\x08\xa5\xab\xeb\xcd\xe2\xbe" +
+	"\xa1\xfc\x30\x2d\xca\xe6\x30\x25\x46\x32\x45\x57\x87\x06\x8e\x29" +
+	"\x10\xf3\x1a\xa4\xaf\x65\xbf\x2e\x5d\x45\x3e\xf2\x46\xae\xef\x4f" +
+	"\x2b\x89\x05\x6e\xc4\xc2\x59\x44\x54\x46\x82\x79\x14\x09\xd9\x0f" +
+	"\x5c\xf9\x41\x1a\x41\xf5\x2b\xab\xc7\xdc\x5b\xbb\xc7\x43\xc7\x96" +
+	"\x69\x27\xb3\x31\x3c\x43\x12\x3c\xdf\xf0\x31\xf7\xd6\xf0\xf1\xd0" +
+	"\xb1\xe1\x48\xb8\xfe\x6c\x2b\x70\x20\xf4\xdb\x2b\xb0\x9f\x87\xdf" +
+	"\x5a\x80\x3f\x25\xf9\x9b\x68\x0f\x0a\x80\x4d\x85\x1c\xc9\x7e\x4c" +
+	"\x4d\xa4\x41\x9b\x70\x62\xf2\xee\x92\x03\x81\xb1\x2e\xc6\xc5\x7a" +
+	"\xbc\xbc\x1b\xfe\xca\xc1\x23\xe6\xc0\xbf\x0b\x24\x33\x57\x47\x43" +
+	"\x31\x51\x03\x8e\xd3\x21\x87\xb6\x18\x22\x67\x9f\x45\x57\xbe\x8a" +
+	"\x6c\x77\xa5\xc8\x35\xf3\x7f\x25\xb2\xf3\xac\x7d\x38\x76\x62\x27" +
+	"\x4c\x03\x0b\xdb\x2d\x1a\x1b\xd9\x85\x68\x26\xf6\xc2\xb9\x0a\x5c" +
+	"\x58\x31\x50\x73\xe0\xb9\xda\x42\x27\x02\x7d\x41\x39\x8f\xf5\x9d" +
+	"\xc2\xf1\x01\xd7\xae\x83\x0f\xfb\xc9\x4f\x40\xec\x94\xfd\x01\x50" +
+	"\xa8\xf1\xab\x88\xb5\xad\xa7\x6c\xda\xcc\xd5\x1c\xd8\x0c\x2a\x91" +
+	"\x93\x10\x2a\xee\xb4\xc4\x34\x68\x28\x78\x46\x88\xd1\xe8\xd6\x08" +
+	"\xfd\x51\xee\x2d\x1b\xb1\xb5\x13\xf9\x27\xfa\xd7\xfd\x8e\x8f\x56" +
+	"\x63\x3b\x76\xaa\x0a\xa4\xdf\x3d\xc8\x3f\xb3\xf7\xdb\x09\x7e\x39" +
+	"\x99\xe8\x6f\xb5\x7e\x5d\xed\x01\x74\x1f\x65\x67\x81\xfd\xa3\xc0" +
+	"\x21\xde\x3f\x8e\x9e\x31\xc1\x7f\xc5\x83\x93\x02\x87\xd3\xfc\xe7" +
+	"\x1e\x1c\x4d\xb8\x63\x3c\xfd\xcf\x21\xe5\x73\x94\xfc\x39\x35\x3a" +
+	"\xda\x0e\x7f\x3b\x4e\x4f\xa3\xe4\xe8\x28\xba\xab\xc6\x0d\x63\x49" +
+	"\x48\x3a\x86\xce\xf6\x22\xe1\x73\x8f\x03\xc1\xfb\x4f\xe4\x3e\xed" +
+	"\x52\x98\xb5\xc0\x2f\x41\x62\x57\xe5\x8f\x60\xd8\xb5\xfe\x1c\x3d" +
+	"\x3d\x53\x3b\xf4\xfc\xbc\x29\x02\x5d\xab\x7f\x4d\x98\x9c\xe1\x4f" +
+	"\xf7\x16\x7f\x89\x1c\xae\x01\xf6\x1b\x52\x28\x49\xff\x7d\x0a\x45" +
+	"\x59\x8c\x6c\x3a\xe6\xb0\x53\x2e\xd2\xdb\x53\xd2\x30\x17\x25\x34" +
+	"\x67\x90\x54\x3a\xb1\xc1\x33\x41\x00\xa5\xbd\x5d\x93\x31\x0d\x51" +
+	"\x43\xa2\xba\xe1\x3e\xd9\x5b\x44\x1e\x8a\x71\xa2\xf4\xe1\xf2\xfd" +
+	"\xdd\x06\x86\x02\x99\x3f\x82\xf0\x6f\x98\x3f\x2c\xe6\xae\xc4\xc4" +
+	"\x9e\xb6\x4b\xd2\x6a\xb5\x67\xdf\x92\xde\xdf\xd7\x9c\xab\xd5\xd6" +
+	"\xf4\x89\xfb\xbd\x77\x92\xe4\x58\x6d\x7d\x05\xb4\x5a\x85\x77\x41" +
+	"\xef\xef\xd0\x90\x56\x2b\x32\x1a\xe3\xa8\x24\x55\x11\xa4\xa3\x14" +
+	"\xd9\x9c\x6d\x83\x2b\xbe\xc8\xc3\x05\xd9\xe1\x8b\xc1\x95\xcf\x03" +
+	"\x6d\x01\x8a\xd8\x0e\x9e\xe4\x45\x7a\xa8\x22\x03\x68\x6b\xd3\x77" +
+	"\x4c\x31\x4e\x28\x94\xba\xd3\x19\x90\x02\xa9\x78\x28\x76\x20\xce" +
+	"\x99\x46\x70\x4f\x2b\xdb\x10\x9e\xd4\x22\x9f\xa7\x27\x0c\xee\x50" +
+	"\x78\x1b\xf1\x4f\x84\x5b\xc1\xc6\xc0\x5c\x21\x83\x11\x67\x24\x7b" +
+	"\x22\x86\xb6\x73\x7d\xa0\x8b\xec\x1d\x4c\x6c\x40\x3c\xfd\xa9\xb2" +
+	"\x8e\x4b\xce\x9e\x36\x63\xca\x7b\xba\x18\x0f\x11\xfd\x74\xad\xfb" +
+	"\x10\x5e\x8b\x90\xa0\xc4\x11\x46\x8e\xa0\xef\xe7\xca\x29\x3f\x23" +
+	"\x41\xc3\x41\xd8\x3f\xd6\x54\x59\x0f\xfc\x82\x8b\xf4\x02\xfe\x83" +
+	"\x4b\xcf\x84\x74\x9e\x33\x3b\x35\xf4\x2e\xff\x94\x2e\x21\xbc\xe3" +
+	"\x3f\x4f\x9f\xb0\xd1\x72\x3a\xc2\x8d\xb2\x33\xa3\xa4\x17\xcc\xf4" +
+	"\x8f\x86\x52\x24\x15\xa1\xe2\x15\x53\x7f\x02\x78\x76\x42\x8c\x3c" +
+	"\x43\x6c\xaf\x27\x21\x81\x7b\xd1\xda\xcc\xa6\xf3\xe7\x8e\x69\xe8" +
+	"\xa6\xeb\x40\x73\x0e\xed\x52\xe4\x93\x0d\x0b\xcf\x90\x73\xc3\x7c" +
+	"\xdb\x9f\xb7\xdf\xae\xef\x77\xb2\x2e\x2e\x45\x3e\x59\xa7\x3f\x11" +
+	"\x25\xa1\x0d\x6c\xa4\x0b\x16\x10\xe1\x1a\xa1\xcc\xbf\x99\x36\xb0" +
+	"\xd5\x4d\xba\x82\xb9\x63\xd1\x95\x25\x70\x99\x2c\x67\xeb\x55\xac" +
+	"\x18\x0f\xfe\xa6\xfa\x4f\x00\x00\x00\xff\xff\xdb\x07\xd3\x5c\xad" +
+	"\x1a\x00\x00"
+
+// DetailedTemplate returns the binary data for a given file.
+func DetailedTemplate() []byte {
+	// This bit of black magic ensures we do not get
+	// unneccesary memcpy's and can read directly from
+	// the .rodata section.
+	var empty [0]byte
+	sx := (*reflect.StringHeader)(unsafe.Pointer(&_DetailedTemplate))
+	b := empty[:]
+	bx := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+	bx.Data = sx.Data
+	bx.Len = len(_DetailedTemplate)
+	bx.Cap = bx.Len
+
+	gz, err := gzip.NewReader(bytes.NewBuffer(b))
+
+	if err != nil {
+		panic("Decompression failed: " + err.Error())
+	}
+
+	var buf bytes.Buffer
+	io.Copy(&buf, gz)
+	gz.Close()
+
+	return buf.Bytes()
+}
--- a/vendor/github.com/JalfResi/justext/htmlRenderer.go
+++ b/vendor/github.com/JalfResi/justext/htmlRenderer.go
@@ -0,0 +1,89 @@
+package justext
+
+import (
+	"fmt"
+	"log"
+	"strings"
+
+	"github.com/levigross/exp-html"
+)
+
+/**
+This should be a separate package!
+And it should be a Writer!
+*/
+
+var selfClosingTags = map[string]bool{
+	"area":     true,
+	"base":     true,
+	"basefont": true,
+	"br":       true,
+	"hr":       true,
+	"input":    true,
+	"img":      true,
+	"link":     true,
+	"meta":     true,
+}
+
+// nodesToString loops over a node tree and generate HTML string
+// Should be moved into html/utils package
+func nodesToString(node *html.Node) string {
+	var response string = ""
+
+	switch node.Type {
+	case html.TextNode:
+		response = html.EscapeString(strings.TrimSpace(node.Data))
+
+	case html.ElementNode, html.DoctypeNode:
+		var att string = ""
+		if len(node.Attr) > 0 {
+			for _, a := range node.Attr {
+				att = fmt.Sprintf("%s %s=\"%s\"", att, a.Key, a.Val)
+			}
+		}
+
+		if _, ok := selfClosingTags[node.Data]; ok {
+			return fmt.Sprintf("<%s%s>", node.Data, att)
+		}
+
+		var content string = ""
+		if len(node.Child) > 0 {
+			for _, n := range node.Child {
+				content = fmt.Sprintf("%s%s", content, nodesToString(n))
+			}
+		}
+		response = fmt.Sprintf("<%s%s>%s</%s>", node.Data, att, content, node.Data)
+
+	case html.DocumentNode:
+		if len(node.Child) > 0 {
+			for _, n := range node.Child {
+				response = nodesToString(n)
+			}
+		}
+
+	case html.CommentNode:
+		// ignore
+
+	default:
+		log.Printf("Unhandled node: %s", nodeTypeToString(node))
+	}
+	return response
+}
+
+func nodeTypeToString(n *html.Node) (t string) {
+	switch n.Type {
+	case html.ErrorNode:
+		t = "Error"
+	case html.TextNode:
+		t = "Text"
+	case html.DocumentNode:
+		t = "Document"
+	case html.ElementNode:
+		t = "Element"
+	case html.CommentNode:
+		t = "Comment"
+	case html.DoctypeNode:
+		t = "Doctype"
+	}
+	return t
+}
--- a/vendor/github.com/JalfResi/justext/paragraphObjectModel.go
+++ b/vendor/github.com/JalfResi/justext/paragraphObjectModel.go
@@ -0,0 +1,160 @@
+package justext
+
+import (
+	"fmt"
+	"github.com/levigross/exp-html"
+	"io"
+	"regexp"
+	"strings"
+)
+
+var (
+	paragraphTags = map[string]bool{
+		"blockquote": true,
+		"caption":    true,
+		"center":     true,
+		"col":        true,
+		"colgroup":   true,
+		"dd":         true,
+		"div":        true,
+		"dl":         true,
+		"dt":         true,
+		"fieldset":   true,
+		"form":       true,
+		"legend":     true,
+		"optgroup":   true,
+		"option":     true,
+		"p":          true,
+		"pre":        true,
+		"table":      true,
+		"td":         true,
+		"textarea":   true,
+		"tfoot":      true,
+		"th":         true,
+		"thead":      true,
+		"tr":         true,
+		"ul":         true,
+		"li":         true,
+		"h1":         true,
+		"h2":         true,
+		"h3":         true,
+		"h4":         true,
+		"h5":         true,
+		"h6":         true,
+	}
+	matchWhiteSpace *regexp.Regexp = regexp.MustCompile("[\n\r\t]+")
+)
+
+type Paragraph struct {
+	DomPath         string
+	TextNodes       []string
+	WordCount       int
+	LinkedCharCount int
+	TagCount        int
+	Text            string
+	StopwordCount   int
+	StopwordDensity float64
+	LinkDensity     float64
+	Heading         bool
+	CfClass         string
+	Class           string
+}
+
+func paragraphObjectModel(htmlStr string) ([]*Paragraph, error) {
+
+	var dom []string
+	var paragraphs []*Paragraph
+	var paragraph *Paragraph = &Paragraph{WordCount: 0, LinkedCharCount: 0, TagCount: 0}
+	var link bool = false
+	var br bool = false
+	var matchToDoErrors *regexp.Regexp = regexp.MustCompile("^html: TODO: ")
+
+	var startNewParagraph func()
+	startNewParagraph = func() {
+		if len(paragraph.TextNodes) != 0 {
+			paragraph.Text = strings.TrimSpace(matchWhiteSpace.ReplaceAllString(strings.Join(paragraph.TextNodes, " "), " "))
+			paragraphs = append(paragraphs, paragraph)
+		}
+		paragraph = &Paragraph{
+			DomPath:         strings.Join(dom, "."),
+			WordCount:       0,
+			LinkedCharCount: 0,
+			TagCount:        0,
+		}
+	}
+
+	z := html.NewTokenizer(strings.NewReader(htmlStr))
+
+	for {
+		tt := z.Next()
+		switch tt {
+
+		case html.ErrorToken:
+			if z.Err() == io.EOF {
+				return paragraphs, nil
+			}
+			if matchToDoErrors.MatchString(fmt.Sprintf("%s", z.Err())) {
+				return nil, z.Err()
+			}
+			continue
+
+		case html.StartTagToken:
+			tmpName, _ := z.TagName()
+			name := string(tmpName)
+			//log.Println("Matched start tag: ", name)
+			dom = append(dom, name)
+			_, ok := paragraphTags[name]
+			if ok || (name == "br" && br) {
+				if name == "br" {
+					paragraph.TagCount--
+				}
+				startNewParagraph()
+			} else {
+				if name == "br" {
+					br = true
+				} else {
+					br = false
+				}
+				if name == "a" {
+					link = true
+				}
+				paragraph.TagCount++
+			}
+
+		case html.EndTagToken:
+			tmpName, _ := z.TagName()
+			name := string(tmpName)
+			//log.Println("Matched end tag: ", name)
+			dom = dom[0 : len(dom)-1]
+			if _, ok := paragraphTags[name]; ok {
+				startNewParagraph()
+			}
+			if name == "a" {
+				link = false
+			}
+
+		case html.TextToken:
+			text := strings.TrimSpace(string(z.Text()))
+			e := 15
+			if len(text) < e {
+				e = len(text)
+			}
+			//log.Println("Matched text: ", text[:e], "...")
+			if text == "" {
+				continue
+			}
+			text = strings.TrimSpace(matchWhiteSpace.ReplaceAllString(text, " "))
+			paragraph.TextNodes = append(paragraph.TextNodes, text)
+			words := strings.Split(text, " ")
+			paragraph.WordCount += len(words)
+			if link {
+				paragraph.LinkedCharCount += len(text)
+			}
+			br = false
+
+		}
+	}
+	startNewParagraph()
+
+	return paragraphs, nil
+}
--- a/vendor/github.com/JalfResi/justext/preprocess.go
+++ b/vendor/github.com/JalfResi/justext/preprocess.go
@@ -0,0 +1,111 @@
+package justext
+
+import (
+	"github.com/levigross/exp-html"
+	"regexp"
+	"strings"
+)
+
+func preprocess(htmlStr, encoding, defaultEncoding, encErrors string) (*html.Node, error) {
+
+	root, err := html.Parse(strings.NewReader(htmlStr))
+	if err != nil {
+		return nil, err
+	}
+
+	addKwTags(root)
+	removeElements(root, []string{"head", "script", "style"})
+
+	return root, nil
+}
+
+type nodeIterator func(n *html.Node)
+
+func nodeIter(n *html.Node, f nodeIterator) {
+	f(n)
+	for _, c := range n.Child {
+		nodeIter(c, f)
+	}
+}
+
+func addKwTags(root *html.Node) *html.Node {
+	var blankText *regexp.Regexp = regexp.MustCompile("^[\n\r\t ]*$")
+	var nodesWithText []*html.Node
+
+	var markTextAndTail nodeIterator
+	markTextAndTail = func(node *html.Node) {
+		if node.Type != html.CommentNode || node.Type != html.DoctypeNode {
+			if node.Type == html.TextNode {
+				nodesWithText = append(nodesWithText, node)
+			}
+		}
+	}
+	nodeIter(root, markTextAndTail)
+
+	for _, node := range nodesWithText {
+		if blankText.MatchString(node.Data) {
+			node.Data = ""
+		} else {
+			kw := &html.Node{
+				Parent: nil,
+				Type:   html.ElementNode,
+				Data:   "kw",
+			}
+			node2 := CopyNode(node, true)
+			kw.Child = append(kw.Child, node2)
+			insertNode(node, kw)
+			node.Parent.Remove(node)
+		}
+	}
+
+	return root
+}
+
+func removeElements(root *html.Node, elementsToRemove []string) {
+	var toBeRemoved []*html.Node
+	var markRemovableNodes = func(node *html.Node) {
+		if node.Type == html.ElementNode {
+			for _, nodeName := range elementsToRemove {
+				if node.Data == nodeName {
+					toBeRemoved = append(toBeRemoved, node)
+				}
+			}
+		}
+	}
+	nodeIter(root, markRemovableNodes)
+
+	for _, node := range toBeRemoved {
+		node.Parent.Remove(node)
+	}
+}
+
+// insertsNode inserts a Node in a Node tree at the position of another node.
+// Should be moved into html/utils package
+func insertNode(originalNode *html.Node, newNode *html.Node) {
+	slice := originalNode.Parent.Child
+	for position, n := range slice {
+		if n == originalNode {
+			originalNode.Parent.Child = append(slice[:position], append([]*html.Node{newNode}, slice[position:]...)...)
+			return
+		}
+	}
+}
+
+func CopyNode(node *html.Node, deep bool) *html.Node {
+	newNode := &html.Node{
+		Type: node.Type,
+		Data: node.Data,
+	}
+
+	if deep && len(node.Child) > 0 {
+		for _, n := range node.Child {
+			newNode.Child = append(newNode.Child, CopyNode(n, true))
+		}
+	}
+
+	for _, i := range node.Attr {
+		newNode.Attr = append(newNode.Attr, html.Attribute{Key: i.Key, Val: i.Val})
+	}
+
+	return newNode
+}
--- a/vendor/github.com/JalfResi/justext/reader.go
+++ b/vendor/github.com/JalfResi/justext/reader.go
@@ -0,0 +1,98 @@
+package justext
+
+import (
+	"errors"
+	"fmt"
+	"github.com/levigross/exp-html"
+	"io"
+	"io/ioutil"
+	"strings"
+)
+
+type Reader struct {
+	LengthLow          int
+	LengthHigh         int
+	Stoplist           map[string]bool
+	StopwordsLow       float64
+	StopwordsHigh      float64
+	MaxLinkDensity     float64
+	MaxHeadingDistance int
+	NoHeadings         bool
+	r                  io.Reader
+}
+
+func NewReader(r io.Reader) *Reader {
+	return &Reader{
+		LengthLow:          70,
+		LengthHigh:         200,
+		StopwordsLow:       0.30,
+		StopwordsHigh:      0.32,
+		MaxLinkDensity:     0.2,
+		MaxHeadingDistance: 200,
+		NoHeadings:         false,
+		r:                  r,
+	}
+}
+
+func (r *Reader) ReadAll() ([]*Paragraph, error) {
+	in, err := ioutil.ReadAll(r.r)
+	if err != nil {
+		return nil, err
+	}
+
+	root, err := preprocess(string(in), "utf-8", "utf-8", "errors")
+	if err != nil {
+		return nil, err
+	}
+	if root == nil {
+		return nil, errors.New("Preprocess has resulted in nil")
+	}
+
+	htmlSource := nodesToString(root)
+	if len(htmlSource) == 0 {
+		return nil, errors.New("MAIN: perprocess has returned an empty string")
+	}
+
+	p, err := paragraphObjectModel(htmlSource)
+	if err != nil {
+		return nil, err
+	}
+	if p == nil {
+		return nil, errors.New("MAIN: P is nil")
+	}
+
+	classifyParagraphs(p, r.Stoplist, r.LengthLow, r.LengthHigh, r.StopwordsLow, r.StopwordsHigh, r.MaxLinkDensity, r.NoHeadings)
+	reviseParagraphClassification(p, r.MaxHeadingDistance)
+
+	return p, nil
+}
+
+func dumpNodes(n *html.Node, tab int, exploreChildNodes bool) string {
+	var childNodes string = ""
+	if exploreChildNodes == true {
+		if len(n.Child) > 0 {
+			for _, c := range n.Child {
+				childNodes = fmt.Sprintf("%s%s\n", childNodes, dumpNodes(c, tab+1, true))
+			}
+		}
+	}
+
+	var t string
+	switch n.Type {
+	case html.ErrorNode:
+		t = "Err"
+	case html.TextNode:
+		t = "T"
+	case html.DocumentNode:
+		t = "D"
+	case html.ElementNode:
+		t = "E"
+	case html.CommentNode:
+		t = "C"
+	case html.DoctypeNode:
+		t = "Dt"
+	}
+
+	tabStr := strings.Repeat(" ", tab)
+	return fmt.Sprintf("%s%s:%s\n%s", tabStr, t, strings.TrimSpace(strings.Replace(n.Data, "\n", "", -1)), childNodes)
+}
--- a/vendor/github.com/JalfResi/justext/reviseParagraphClassification.go
+++ b/vendor/github.com/JalfResi/justext/reviseParagraphClassification.go
@@ -0,0 +1,118 @@
+package justext
+
+// Context-sensitive paragraph classification. Assumes that classify_pragraphs has already been called.
+func reviseParagraphClassification(paragraphs []*Paragraph, maxHeadingDistance int) {
+
+	// Copy classes
+	for _, paragraph := range paragraphs {
+		paragraph.Class = paragraph.CfClass
+	}
+
+	// Good headings
+	var j int = 0
+	var distance int
+	for i, paragraph := range paragraphs {
+		if !(paragraph.Heading && paragraph.Class == "short") {
+			continue
+		}
+
+		j = i + 1
+		distance = 0
+
+		for j < len(paragraphs) && distance <= maxHeadingDistance {
+			if paragraphs[j].Class == "good" {
+				paragraph.Class = "neargood"
+				break
+			}
+			distance += len(paragraphs[j].Text)
+			j += 1
+		}
+	}
+
+	// Classify short
+	var newClasses []string = make([]string, len(paragraphs))
+	for i, paragraph := range paragraphs {
+		if paragraph.Class != "short" {
+			continue
+		}
+
+		var prevNeighbour string = getPrevNeighbour(i, paragraphs, true)
+		var nextNeighbour string = getNextNeighbour(i, paragraphs, true)
+
+		var neighbours map[string]bool = make(map[string]bool)
+		neighbours[prevNeighbour] = true
+		neighbours[nextNeighbour] = true
+
+		if _, ok := neighbours["good"]; ok && len(neighbours) == 1 {
+			newClasses[i] = "good"
+		} else if _, ok := neighbours["bad"]; ok && len(neighbours) == 1 {
+			newClasses[i] = "bad"
+			// neighbours must contain both good and bad
+		} else if (prevNeighbour == "bad" && getPrevNeighbour(i, paragraphs, false) == "neargood") || (nextNeighbour == "bad" && getNextNeighbour(i, paragraphs, false) == "neargood") {
+			newClasses[i] = "good"
+		} else {
+			newClasses[i] = "bad"
+		}
+	}
+
+	for i, c := range newClasses {
+		if c != "" {
+			paragraphs[i].Class = c
+		}
+	}
+
+	// revise neargood
+	for i, paragraph := range paragraphs {
+		if paragraph.Class != "neargood" {
+			continue
+		}
+
+		var prevNeighbour string = getPrevNeighbour(i, paragraphs, true)
+		var nextNeighbour string = getNextNeighbour(i, paragraphs, true)
+
+		if prevNeighbour == "bad" && nextNeighbour == "bad" {
+			paragraph.Class = "bad"
+		} else {
+			paragraph.Class = "good"
+		}
+	}
+
+	// more good headings
+	for i, paragraph := range paragraphs {
+		if !(paragraph.Heading && paragraph.Class == "bad" && paragraph.CfClass != "bad") {
+			continue
+		}
+		j = i + 1
+		distance = 0
+		for j < len(paragraphs) && distance <= maxHeadingDistance {
+			if paragraphs[j].Class == "good" {
+				paragraph.Class = "good"
+				break
+			}
+			distance += len(paragraphs[j].Text)
+			j += 1
+		}
+	}
+}
+
+func getPrevNeighbour(i int, paragraphs []*Paragraph, ignoreNeargood bool) string {
+	return getNeighbour(i, paragraphs, ignoreNeargood, -1, -1)
+}
+
+func getNextNeighbour(i int, paragraphs []*Paragraph, ignoreNeargood bool) string {
+	return getNeighbour(i, paragraphs, ignoreNeargood, 1, len(paragraphs))
+}
+
+func getNeighbour(i int, paragraphs []*Paragraph, ignoreNeargood bool, inc int, boundary int) string {
+	for i+inc != boundary {
+		i += inc
+		var c string = paragraphs[i].Class
+		if c == "good" || c == "bad" {
+			return c
+		}
+		if c == "neargood" && !ignoreNeargood {
+			return c
+		}
+	}
+	return "bad"
+}
--- a/vendor/github.com/JalfResi/justext/stoplists.go
+++ b/vendor/github.com/JalfResi/justext/stoplists.go
@@ -0,0 +1,158 @@
+package justext
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io/ioutil"
+)
+
+type ResourceFunc func() ([]byte, error)
+
+var stoplists = map[string]ResourceFunc{}
+
+func RegisterStoplist(name string, resourceFunc ResourceFunc) {
+	stoplists[name] = resourceFunc
+}
+
+/*
+{
+	"Afrikaans":               AfrikaansStoplist,
+	"Albanian":                AlbanianStoplist,
+	"Arabic":                  ArabicStoplist,
+	"Aragonese":               AragoneseStoplist,
+	"Armenian":                ArmenianStoplist,
+	"Aromanian":               AromanianStoplist,
+	"Asturian":                AsturianStoplist,
+	"Azerbaijani":             AzerbaijaniStoplist,
+	"Basque":                  BasqueStoplist,
+	"Belarusian":              BelarusianStoplist,
+	"Belarusian_Taraskievica": Belarusian_TaraskievicaStoplist,
+	"Bengali":                 BengaliStoplist,
+	"Bishnupriya_Manipuri":    Bishnupriya_ManipuriStoplist,
+	"Bosnian":                 BosnianStoplist,
+	"Breton":                  BretonStoplist,
+	"Bulgarian":               BulgarianStoplist,
+	"Catalan":                 CatalanStoplist,
+	"Cebuano":                 CebuanoStoplist,
+	"Chuvash":                 ChuvashStoplist,
+	"Croatian":                CroatianStoplist,
+	"Czech":                   CzechStoplist,
+	"Danish":                  DanishStoplist,
+	"Dutch":                   DutchStoplist,
+	"English":                 EnglishStoplist,
+	"Esperanto":               EsperantoStoplist,
+	"Estonian":                EstonianStoplist,
+	"Finnish":                 FinnishStoplist,
+	"French":                  FrenchStoplist,
+	"Galician":                GalicianStoplist,
+	"Georgian":                GeorgianStoplist,
+	"German":                  GermanStoplist,
+	"Greek":                   GreekStoplist,
+	"Gujarati":                GujaratiStoplist,
+	"Haitian":                 HaitianStoplist,
+	"Hebrew":                  HebrewStoplist,
+	"Hindi":                   HindiStoplist,
+	"Hungarian":               HungarianStoplist,
+	"Icelandic":               IcelandicStoplist,
+	"Ido":                     IdoStoplist,
+	"Igbo":                    IgboStoplist,
+	"Indonesian":              IndonesianStoplist,
+	"Irish":                   IrishStoplist,
+	"Italian":                 ItalianStoplist,
+	"Javanese":                JavaneseStoplist,
+	"Kannada":                 KannadaStoplist,
+	"Kazakh":                  KazakhStoplist,
+	"Korean":                  KoreanStoplist,
+	"Kurdish":                 KurdishStoplist,
+	"Kyrgyz":                  KyrgyzStoplist,
+	"Latin":                   LatinStoplist,
+	"Latvian":                 LatvianStoplist,
+	"Lithuanian":              LithuanianStoplist,
+	"Lombard":                 LombardStoplist,
+	"Low_Saxon":               Low_SaxonStoplist,
+	"Luxembourgish":           LuxembourgishStoplist,
+	"Macedonian":              MacedonianStoplist,
+	"Malay":                   MalayStoplist,
+	"Malayalam":               MalayalamStoplist,
+	"Maltese":                 MalteseStoplist,
+	"Marathi":                 MarathiStoplist,
+	"Neapolitan":              NeapolitanStoplist,
+	"Nepali":                  NepaliStoplist,
+	"Newar":                   NewarStoplist,
+	"Norwegian_Bokmal":        Norwegian_BokmalStoplist,
+	"Norwegian_Nynorsk":       Norwegian_NynorskStoplist,
+	"Occitan":                 OccitanStoplist,
+	"Persian":                 PersianStoplist,
+	"Piedmontese":             PiedmonteseStoplist,
+	"Polish":                  PolishStoplist,
+	"Portuguese":              PortugueseStoplist,
+	"Quechua":                 QuechuaStoplist,
+	"Romanian":                RomanianStoplist,
+	"Russian":                 RussianStoplist,
+	"Samogitian":              SamogitianStoplist,
+	"Serbian":                 SerbianStoplist,
+	"Serbo_Croatian":          Serbo_CroatianStoplist,
+	"Sicilian":                SicilianStoplist,
+	"Simple_English":          Simple_EnglishStoplist,
+	"Slovak":                  SlovakStoplist,
+	"Slovenian":               SlovenianStoplist,
+	"Spanish":                 SpanishStoplist,
+	"Sundanese":               SundaneseStoplist,
+	"Swahili":                 SwahiliStoplist,
+	"Swedish":                 SwedishStoplist,
+	"Tagalog":                 TagalogStoplist,
+	"Tamil":                   TamilStoplist,
+	"Telugu":                  TeluguStoplist,
+	"Turkish":                 TurkishStoplist,
+	"Turkmen":                 TurkmenStoplist,
+	"Ukrainian":               UkrainianStoplist,
+	"Urdu":                    UrduStoplist,
+	"Uzbek":                   UzbekStoplist,
+	"Vietnamese":              VietnameseStoplist,
+	"Volapuk":                 VolapukStoplist,
+	"Walloon":                 WalloonStoplist,
+	"Waray_Waray":             Waray_WarayStoplist,
+	"Welsh":                   WelshStoplist,
+	"West_Frisian":            West_FrisianStoplist,
+	"Western_Panjabi":         Western_PanjabiStoplist,
+	"Yoruba":                  YorubaStoplist,
+}
+*/
+
+func ReadStoplist(filename string) (map[string]bool, error) {
+	data, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+
+	db := bytes.Split(data, []uint8("\n"))
+
+	// Convert to map
+	var list = make(map[string]bool)
+	for _, val := range db {
+		list[string(val)] = true
+	}
+
+	return list, nil
+}
+
+func GetStoplist(language string) (map[string]bool, error) {
+	if _, ok := stoplists[language]; !ok {
+		return nil, errors.New(fmt.Sprintf("Language %s not supported", language))
+	}
+
+	data, err := stoplists[language]()
+	if err != nil {
+		return nil, err
+	}
+	db := bytes.Split(data, []uint8("\n"))
+
+	// Convert to map
+	var list = make(map[string]bool)
+	for _, val := range db {
+		list[string(val)] = true
+	}
+
+	return list, nil
+}
--- a/vendor/github.com/JalfResi/justext/writer.go
+++ b/vendor/github.com/JalfResi/justext/writer.go
@@ -0,0 +1,161 @@
+package justext
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"strings"
+	"text/template"
+)
+
+// NOTE:
+// Make a new type:
+//  type JusText []paragraphs
+
+const (
+	MODE_DEFAULT  = 1
+	MODE_DETAILED = 2
+)
+
+type Writer struct {
+	Mode          int
+	NoBoilerplate bool
+	Stoplist      map[string]bool
+	w             io.Writer
+}
+
+func NewWriter(w io.Writer) *Writer {
+	return &Writer{
+		Mode:          MODE_DEFAULT,
+		NoBoilerplate: true,
+		w:             w,
+	}
+}
+
+func (w *Writer) WriteAll(paragraphs []*Paragraph) error {
+	switch w.Mode {
+	case MODE_DEFAULT:
+		return w.outputDefault(paragraphs)
+		break
+
+	case MODE_DETAILED:
+		return w.outputDetailed(paragraphs)
+		break
+
+	default:
+		return errors.New("Unrecognised mode")
+	}
+
+	return nil
+}
+
+func IsGood(args ...interface{}) (result bool) {
+	result = true
+	for _, val := range args {
+		if val != "good" {
+			result = false
+			return
+		}
+	}
+	return
+}
+
+func (w *Writer) outputDefault(paragraphs []*Paragraph) error {
+	templateData := DefaultTemplate()
+	t := template.New("default")
+	t.Funcs(template.FuncMap{"TrimSpace": strings.TrimSpace})
+	t.Funcs(template.FuncMap{"IsGood": IsGood})
+
+	templ, err := t.Parse(string(templateData))
+	if err != nil {
+		return err
+	}
+
+	var data = struct {
+		Paragraphs    []*Paragraph
+		NoBoilerplate bool
+	}{paragraphs, w.NoBoilerplate}
+
+	return templ.Execute(w.w, data)
+}
+
+func (w *Writer) outputDetailed(paragraphs []*Paragraph) error {
+	templateData := DetailedTemplate()
+	var markStopwords func(args ...interface{}) string
+	markStopwords = func(args ...interface{}) string {
+
+		var output string = ""
+		words := strings.Split(args[0].(string), " ")
+		for _, word := range words {
+			if _, ok := w.Stoplist[strings.TrimSpace(word)]; ok {
+				output = fmt.Sprintf("%s<span class=\"stopword\">%s</span> ", output, word)
+			} else {
+				output = fmt.Sprintf("%s%s ", output, word)
+			}
+		}
+
+		return output
+	}
+
+	t := template.New("detailed")
+	t.Funcs(template.FuncMap{"TrimSpace": strings.TrimSpace})
+	t.Funcs(template.FuncMap{"MarkStopwords": markStopwords})
+
+	templ, err := t.Parse(string(templateData))
+	if err != nil {
+		return err
+	}
+
+	var data = struct {
+		Paragraphs []*Paragraph
+	}{paragraphs}
+
+	return templ.Execute(w.w, data)
+}
+
+func (w *Writer) OutputDebug(paragraphs []*Paragraph) {
+	for _, paragraph := range paragraphs {
+		log.Println(paragraph.DomPath)
+		log.Println("\tfinal class: ", paragraph.Class)
+		log.Println("\tcontext-free class: ", paragraph.CfClass)
+		log.Println("\theading: ", paragraph.Heading)
+		log.Println("\tlength (in characters): ", len(paragraph.Text))
+		log.Println("\tnumber of characters with links: ", paragraph.LinkedCharCount)
+		log.Println("\tlink density: ", paragraph.LinkDensity)
+		log.Println("\tnumber of words: ", paragraph.WordCount)
+		log.Println("\tnumber of stop words: ", paragraph.StopwordCount)
+		log.Println("\tstop word density: ", paragraph.StopwordDensity)
+	}
+}
+
+// TO-DO:
+// Need an output feature that returns a de-duped space separated text file of all the
+// words in the output document sans-boilerplate. Also needs option to exclude stoplist
+// words from that output too.
+
+// TO-DO:
+// Need an output feature that returns the content of a stop list (or do we just make
+// the function getStoplist public? Might be a lot easier...)
+
+/*
+func (w *Writer) outputKrdwrd(paragraphs []*Paragraph) (output string) {
+	for _, paragraph := range paragraphs {
+		var cls int
+		if paragraph.Class == "good" || paragraph.Class == "neargood" {
+			if paragraph.Heading {
+				cls = 2
+			} else {
+				cls = 3
+			}
+		} else {
+			cls = 1
+		}
+		for _, textNode := range paragraph.TextNodes {
+			output = fmt.Sprintf("%s%i\t%s", output, cls, strings.TrimSpace(textNode))
+		}
+	}
+
+	return output
+}
+*/
--- a/vendor/github.com/PuerkitoBio/goquery/.gitattributes
+++ b/vendor/github.com/PuerkitoBio/goquery/.gitattributes
@@ -0,0 +1 @@
+testdata/* linguist-vendored
--- a/vendor/github.com/PuerkitoBio/goquery/.gitignore
+++ b/vendor/github.com/PuerkitoBio/goquery/.gitignore
@@ -0,0 +1,16 @@
+# editor temporary files
+*.sublime-*
+.DS_Store
+*.swp
+#*.*#
+tags
+
+# direnv config
+.env*
+
+# test binaries
+*.test
+
+# coverage and profilte outputs
+*.out
+
--- a/vendor/github.com/PuerkitoBio/goquery/.travis.yml
+++ b/vendor/github.com/PuerkitoBio/goquery/.travis.yml
@@ -0,0 +1,17 @@
+language: go
+
+go:
+    - 1.2.x
+    - 1.3.x
+    - 1.4.x
+    - 1.5.x
+    - 1.6.x
+    - 1.7.x
+    - 1.8.x
+    - 1.9.x
+    - 1.10.x
+    - 1.11.x
+    - 1.12.x
+    - 1.13.x
+    - tip
+
--- a/vendor/github.com/PuerkitoBio/goquery/LICENSE
+++ b/vendor/github.com/PuerkitoBio/goquery/LICENSE
@@ -0,0 +1,12 @@
+Copyright (c) 2012-2016, Martin Angers & Contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+* Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
--- a/vendor/github.com/PuerkitoBio/goquery/README.md
+++ b/vendor/github.com/PuerkitoBio/goquery/README.md
@@ -0,0 +1,183 @@
+# goquery - a little like that j-thing, only in Go
+[![build status](https://secure.travis-ci.org/PuerkitoBio/goquery.svg?branch=master)](http://travis-ci.org/PuerkitoBio/goquery) [![GoDoc](https://godoc.org/github.com/PuerkitoBio/goquery?status.png)](http://godoc.org/github.com/PuerkitoBio/goquery) [![Sourcegraph Badge](https://sourcegraph.com/github.com/PuerkitoBio/goquery/-/badge.svg)](https://sourcegraph.com/github.com/PuerkitoBio/goquery?badge)
+
+goquery brings a syntax and a set of features similar to [jQuery][] to the [Go language][go]. It is based on Go's [net/html package][html] and the CSS Selector library [cascadia][]. Since the net/html parser returns nodes, and not a full-featured DOM tree, jQuery's stateful manipulation functions (like height(), css(), detach()) have been left off.
+
+Also, because the net/html parser requires UTF-8 encoding, so does goquery: it is the caller's responsibility to ensure that the source document provides UTF-8 encoded HTML. See the [wiki][] for various options to do this.
+
+Syntax-wise, it is as close as possible to jQuery, with the same function names when possible, and that warm and fuzzy chainable interface. jQuery being the ultra-popular library that it is, I felt that writing a similar HTML-manipulating library was better to follow its API than to start anew (in the same spirit as Go's `fmt` package), even though some of its methods are less than intuitive (looking at you, [index()][index]...).
+
+## Table of Contents
+
+* [Installation](#installation)
+* [Changelog](#changelog)
+* [API](#api)
+* [Examples](#examples)
+* [Related Projects](#related-projects)
+* [Support](#support)
+* [License](#license)
+
+## Installation
+
+Please note that because of the net/html dependency, goquery requires Go1.1+.
+
+    $ go get github.com/PuerkitoBio/goquery
+
+(optional) To run unit tests:
+
+    $ cd $GOPATH/src/github.com/PuerkitoBio/goquery
+    $ go test
+
+(optional) To run benchmarks (warning: it runs for a few minutes):
+
+    $ cd $GOPATH/src/github.com/PuerkitoBio/goquery
+    $ go test -bench=".*"
+
+## Changelog
+
+**Note that goquery's API is now stable, and will not break.**
+
+*    **2020-02-04 (v1.5.1)** : Update module dependencies.
+*    **2018-11-15 (v1.5.0)** : Go module support (thanks @Zaba505).
+*    **2018-06-07 (v1.4.1)** : Add `NewDocumentFromReader` examples.
+*    **2018-03-24 (v1.4.0)** : Deprecate `NewDocument(url)` and `NewDocumentFromResponse(response)`.
+*    **2018-01-28 (v1.3.0)** : Add `ToEnd` constant to `Slice` until the end of the selection (thanks to @davidjwilkins for raising the issue).
+*    **2018-01-11 (v1.2.0)** : Add `AddBack*` and deprecate `AndSelf` (thanks to @davidjwilkins).
+*    **2017-02-12 (v1.1.0)** : Add `SetHtml` and `SetText` (thanks to @glebtv).
+*    **2016-12-29 (v1.0.2)** : Optimize allocations for `Selection.Text` (thanks to @radovskyb).
+*    **2016-08-28 (v1.0.1)** : Optimize performance for large documents.
+*    **2016-07-27 (v1.0.0)** : Tag version 1.0.0.
+*    **2016-06-15** : Invalid selector strings internally compile to a `Matcher` implementation that never matches any node (instead of a panic). So for example, `doc.Find("~")` returns an empty `*Selection` object.
+*    **2016-02-02** : Add `NodeName` utility function similar to the DOM's `nodeName` property. It returns the tag name of the first element in a selection, and other relevant values of non-element nodes (see godoc for details). Add `OuterHtml` utility function similar to the DOM's `outerHTML` property (named `OuterHtml` in small caps for consistency with the existing `Html` method on the `Selection`).
+*    **2015-04-20** : Add `AttrOr` helper method to return the attribute's value or a default value if absent. Thanks to [piotrkowalczuk][piotr].
+*    **2015-02-04** : Add more manipulation functions - Prepend* - thanks again to [Andrew Stone][thatguystone].
+*    **2014-11-28** : Add more manipulation functions - ReplaceWith*, Wrap* and Unwrap - thanks again to [Andrew Stone][thatguystone].
+*    **2014-11-07** : Add manipulation functions (thanks to [Andrew Stone][thatguystone]) and `*Matcher` functions, that receive compiled cascadia selectors instead of selector strings, thus avoiding potential panics thrown by goquery via `cascadia.MustCompile` calls. This results in better performance (selectors can be compiled once and reused) and more idiomatic error handling (you can handle cascadia's compilation errors, instead of recovering from panics, which had been bugging me for a long time). Note that the actual type expected is a `Matcher` interface, that `cascadia.Selector` implements. Other matcher implementations could be used.
+*    **2014-11-06** : Change import paths of net/html to golang.org/x/net/html (see https://groups.google.com/forum/#!topic/golang-nuts/eD8dh3T9yyA). Make sure to update your code to use the new import path too when you call goquery with `html.Node`s.
+*    **v0.3.2** : Add `NewDocumentFromReader()` (thanks jweir) which allows creating a goquery document from an io.Reader.
+*    **v0.3.1** : Add `NewDocumentFromResponse()` (thanks assassingj) which allows creating a goquery document from an http response.
+*    **v0.3.0** : Add `EachWithBreak()` which allows to break out of an `Each()` loop by returning false. This function was added instead of changing the existing `Each()` to avoid breaking compatibility.
+*    **v0.2.1** : Make go-getable, now that [go.net/html is Go1.0-compatible][gonet] (thanks to @matrixik for pointing this out).
+*    **v0.2.0** : Add support for negative indices in Slice(). **BREAKING CHANGE** `Document.Root` is removed, `Document` is now a `Selection` itself (a selection of one, the root element, just like `Document.Root` was before). Add jQuery's Closest() method.
+*    **v0.1.1** : Add benchmarks to use as baseline for refactorings, refactor Next...() and Prev...() methods to use the new html package's linked list features (Next/PrevSibling, FirstChild). Good performance boost (40+% in some cases).
+*    **v0.1.0** : Initial release.
+
+## API
+
+goquery exposes two structs, `Document` and `Selection`, and the `Matcher` interface. Unlike jQuery, which is loaded as part of a DOM document, and thus acts on its containing document, goquery doesn't know which HTML document to act upon. So it needs to be told, and that's what the `Document` type is for. It holds the root document node as the initial Selection value to manipulate.
+
+jQuery often has many variants for the same function (no argument, a selector string argument, a jQuery object argument, a DOM element argument, ...). Instead of exposing the same features in goquery as a single method with variadic empty interface arguments, statically-typed signatures are used following this naming convention:
+
+*    When the jQuery equivalent can be called with no argument, it has the same name as jQuery for the no argument signature (e.g.: `Prev()`), and the version with a selector string argument is called `XxxFiltered()` (e.g.: `PrevFiltered()`)
+*    When the jQuery equivalent **requires** one argument, the same name as jQuery is used for the selector string version (e.g.: `Is()`)
+*    The signatures accepting a jQuery object as argument are defined in goquery as `XxxSelection()` and take a `*Selection` object as argument (e.g.: `FilterSelection()`)
+*    The signatures accepting a DOM element as argument in jQuery are defined in goquery as `XxxNodes()` and take a variadic argument of type `*html.Node` (e.g.: `FilterNodes()`)
+*    The signatures accepting a function as argument in jQuery are defined in goquery as `XxxFunction()` and take a function as argument (e.g.: `FilterFunction()`)
+*    The goquery methods that can be called with a selector string have a corresponding version that take a `Matcher` interface and are defined as `XxxMatcher()` (e.g.: `IsMatcher()`)
+
+Utility functions that are not in jQuery but are useful in Go are implemented as functions (that take a `*Selection` as parameter), to avoid a potential naming clash on the `*Selection`'s methods (reserved for jQuery-equivalent behaviour).
+
+The complete [godoc reference documentation can be found here][doc].
+
+Please note that Cascadia's selectors do not necessarily match all supported selectors of jQuery (Sizzle). See the [cascadia project][cascadia] for details. Invalid selector strings compile to a `Matcher` that fails to match any node. Behaviour of the various functions that take a selector string as argument follows from that fact, e.g. (where `~` is an invalid selector string):
+
+* `Find("~")` returns an empty selection because the selector string doesn't match anything.
+* `Add("~")` returns a new selection that holds the same nodes as the original selection, because it didn't add any node (selector string didn't match anything).
+* `ParentsFiltered("~")` returns an empty selection because the selector string doesn't match anything.
+* `ParentsUntil("~")` returns all parents of the selection because the selector string didn't match any element to stop before the top element.
+
+## Examples
+
+See some tips and tricks in the [wiki][].
+
+Adapted from example_test.go:
+
+```Go
+package main
+
+import (
+  "fmt"
+  "log"
+  "net/http"
+
+  "github.com/PuerkitoBio/goquery"
+)
+
+func ExampleScrape() {
+  // Request the HTML page.
+  res, err := http.Get("http://metalsucks.net")
+  if err != nil {
+    log.Fatal(err)
+  }
+  defer res.Body.Close()
+  if res.StatusCode != 200 {
+    log.Fatalf("status code error: %d %s", res.StatusCode, res.Status)
+  }
+
+  // Load the HTML document
+  doc, err := goquery.NewDocumentFromReader(res.Body)
+  if err != nil {
+    log.Fatal(err)
+  }
+
+  // Find the review items
+  doc.Find(".sidebar-reviews article .content-block").Each(func(i int, s *goquery.Selection) {
+    // For each item found, get the band and title
+    band := s.Find("a").Text()
+    title := s.Find("i").Text()
+    fmt.Printf("Review %d: %s - %s\n", i, band, title)
+  })
+}
+
+func main() {
+  ExampleScrape()
+}
+```
+
+## Related Projects
+
+- [Goq][goq], an HTML deserialization and scraping library based on goquery and struct tags.
+- [andybalholm/cascadia][cascadia], the CSS selector library used by goquery.
+- [suntong/cascadia][cascadiacli], a command-line interface to the cascadia CSS selector library, useful to test selectors.
+- [gocolly/colly](https://github.com/gocolly/colly), a lightning fast and elegant Scraping Framework
+- [gnulnx/goperf](https://github.com/gnulnx/goperf), a website performance test tool that also fetches static assets.
+- [MontFerret/ferret](https://github.com/MontFerret/ferret), declarative web scraping.
+- [tacusci/berrycms](https://github.com/tacusci/berrycms), a modern simple to use CMS with easy to write plugins
+- [Dataflow kit](https://github.com/slotix/dataflowkit), Web Scraping framework for Gophers.  
+- [Geziyor](https://github.com/geziyor/geziyor), a fast web crawling & scraping framework for Go. Supports JS rendering.
+
+## Support
+
+There are a number of ways you can support the project:
+
+* Use it, star it, build something with it, spread the word!
+  - If you do build something open-source or otherwise publicly-visible, let me know so I can add it to the [Related Projects](#related-projects) section!
+* Raise issues to improve the project (note: doc typos and clarifications are issues too!)
+  - Please search existing issues before opening a new one - it may have already been adressed.
+* Pull requests: please discuss new code in an issue first, unless the fix is really trivial.
+  - Make sure new code is tested.
+  - Be mindful of existing code - PRs that break existing code have a high probability of being declined, unless it fixes a serious issue.
+
+If you desperately want to send money my way, I have a BuyMeACoffee.com page:
+
+<a href="https://www.buymeacoffee.com/mna" target="_blank"><img src="https://www.buymeacoffee.com/assets/img/custom_images/orange_img.png" alt="Buy Me A Coffee" style="height: 41px !important;width: 174px !important;box-shadow: 0px 3px 2px 0px rgba(190, 190, 190, 0.5) !important;-webkit-box-shadow: 0px 3px 2px 0px rgba(190, 190, 190, 0.5) !important;" ></a>
+
+## License
+
+The [BSD 3-Clause license][bsd], the same as the [Go language][golic]. Cascadia's license is [here][caslic].
+
+[jquery]: http://jquery.com/
+[go]: http://golang.org/
+[cascadia]: https://github.com/andybalholm/cascadia
+[cascadiacli]: https://github.com/suntong/cascadia
+[bsd]: http://opensource.org/licenses/BSD-3-Clause
+[golic]: http://golang.org/LICENSE
+[caslic]: https://github.com/andybalholm/cascadia/blob/master/LICENSE
+[doc]: http://godoc.org/github.com/PuerkitoBio/goquery
+[index]: http://api.jquery.com/index/
+[gonet]: https://github.com/golang/net/
+[html]: http://godoc.org/golang.org/x/net/html
+[wiki]: https://github.com/PuerkitoBio/goquery/wiki/Tips-and-tricks
+[thatguystone]: https://github.com/thatguystone
+[piotr]: https://github.com/piotrkowalczuk
+[goq]: https://github.com/andrewstuart/goq
--- a/vendor/github.com/PuerkitoBio/goquery/array.go
+++ b/vendor/github.com/PuerkitoBio/goquery/array.go
@@ -0,0 +1,124 @@
+package goquery
+
+import (
+	"golang.org/x/net/html"
+)
+
+const (
+	maxUint = ^uint(0)
+	maxInt  = int(maxUint >> 1)
+
+	// ToEnd is a special index value that can be used as end index in a call
+	// to Slice so that all elements are selected until the end of the Selection.
+	// It is equivalent to passing (*Selection).Length().
+	ToEnd = maxInt
+)
+
+// First reduces the set of matched elements to the first in the set.
+// It returns a new Selection object, and an empty Selection object if the
+// the selection is empty.
+func (s *Selection) First() *Selection {
+	return s.Eq(0)
+}
+
+// Last reduces the set of matched elements to the last in the set.
+// It returns a new Selection object, and an empty Selection object if
+// the selection is empty.
+func (s *Selection) Last() *Selection {
+	return s.Eq(-1)
+}
+
+// Eq reduces the set of matched elements to the one at the specified index.
+// If a negative index is given, it counts backwards starting at the end of the
+// set. It returns a new Selection object, and an empty Selection object if the
+// index is invalid.
+func (s *Selection) Eq(index int) *Selection {
+	if index < 0 {
+		index += len(s.Nodes)
+	}
+
+	if index >= len(s.Nodes) || index < 0 {
+		return newEmptySelection(s.document)
+	}
+
+	return s.Slice(index, index+1)
+}
+
+// Slice reduces the set of matched elements to a subset specified by a range
+// of indices. The start index is 0-based and indicates the index of the first
+// element to select. The end index is 0-based and indicates the index at which
+// the elements stop being selected (the end index is not selected).
+//
+// The indices may be negative, in which case they represent an offset from the
+// end of the selection.
+//
+// The special value ToEnd may be specified as end index, in which case all elements
+// until the end are selected. This works both for a positive and negative start
+// index.
+func (s *Selection) Slice(start, end int) *Selection {
+	if start < 0 {
+		start += len(s.Nodes)
+	}
+	if end == ToEnd {
+		end = len(s.Nodes)
+	} else if end < 0 {
+		end += len(s.Nodes)
+	}
+	return pushStack(s, s.Nodes[start:end])
+}
+
+// Get retrieves the underlying node at the specified index.
+// Get without parameter is not implemented, since the node array is available
+// on the Selection object.
+func (s *Selection) Get(index int) *html.Node {
+	if index < 0 {
+		index += len(s.Nodes) // Negative index gets from the end
+	}
+	return s.Nodes[index]
+}
+
+// Index returns the position of the first element within the Selection object
+// relative to its sibling elements.
+func (s *Selection) Index() int {
+	if len(s.Nodes) > 0 {
+		return newSingleSelection(s.Nodes[0], s.document).PrevAll().Length()
+	}
+	return -1
+}
+
+// IndexSelector returns the position of the first element within the
+// Selection object relative to the elements matched by the selector, or -1 if
+// not found.
+func (s *Selection) IndexSelector(selector string) int {
+	if len(s.Nodes) > 0 {
+		sel := s.document.Find(selector)
+		return indexInSlice(sel.Nodes, s.Nodes[0])
+	}
+	return -1
+}
+
+// IndexMatcher returns the position of the first element within the
+// Selection object relative to the elements matched by the matcher, or -1 if
+// not found.
+func (s *Selection) IndexMatcher(m Matcher) int {
+	if len(s.Nodes) > 0 {
+		sel := s.document.FindMatcher(m)
+		return indexInSlice(sel.Nodes, s.Nodes[0])
+	}
+	return -1
+}
+
+// IndexOfNode returns the position of the specified node within the Selection
+// object, or -1 if not found.
+func (s *Selection) IndexOfNode(node *html.Node) int {
+	return indexInSlice(s.Nodes, node)
+}
+
+// IndexOfSelection returns the position of the first node in the specified
+// Selection object within this Selection object, or -1 if not found.
+func (s *Selection) IndexOfSelection(sel *Selection) int {
+	if sel != nil && len(sel.Nodes) > 0 {
+		return indexInSlice(s.Nodes, sel.Nodes[0])
+	}
+	return -1
+}
--- a/vendor/github.com/PuerkitoBio/goquery/doc.go
+++ b/vendor/github.com/PuerkitoBio/goquery/doc.go
@@ -0,0 +1,123 @@
+// Copyright (c) 2012-2016, Martin Angers & Contributors
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification,
+// are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation and/or
+// other materials provided with the distribution.
+// * Neither the name of the author nor the names of its contributors may be used to
+// endorse or promote products derived from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS
+// OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY
+// WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+/*
+Package goquery implements features similar to jQuery, including the chainable
+syntax, to manipulate and query an HTML document.
+
+It brings a syntax and a set of features similar to jQuery to the Go language.
+It is based on Go's net/html package and the CSS Selector library cascadia.
+Since the net/html parser returns nodes, and not a full-featured DOM
+tree, jQuery's stateful manipulation functions (like height(), css(), detach())
+have been left off.
+
+Also, because the net/html parser requires UTF-8 encoding, so does goquery: it is
+the caller's responsibility to ensure that the source document provides UTF-8 encoded HTML.
+See the repository's wiki for various options on how to do this.
+
+Syntax-wise, it is as close as possible to jQuery, with the same method names when
+possible, and that warm and fuzzy chainable interface. jQuery being the
+ultra-popular library that it is, writing a similar HTML-manipulating
+library was better to follow its API than to start anew (in the same spirit as
+Go's fmt package), even though some of its methods are less than intuitive (looking
+at you, index()...).
+
+It is hosted on GitHub, along with additional documentation in the README.md
+file: https://github.com/puerkitobio/goquery
+
+Please note that because of the net/html dependency, goquery requires Go1.1+.
+
+The various methods are split into files based on the category of behavior.
+The three dots (...) indicate that various "overloads" are available.
+
+* array.go : array-like positional manipulation of the selection.
+    - Eq()
+    - First()
+    - Get()
+    - Index...()
+    - Last()
+    - Slice()
+
+* expand.go : methods that expand or augment the selection's set.
+    - Add...()
+    - AndSelf()
+    - Union(), which is an alias for AddSelection()
+
+* filter.go : filtering methods, that reduce the selection's set.
+    - End()
+    - Filter...()
+    - Has...()
+    - Intersection(), which is an alias of FilterSelection()
+    - Not...()
+
+* iteration.go : methods to loop over the selection's nodes.
+    - Each()
+    - EachWithBreak()
+    - Map()
+
+* manipulation.go : methods for modifying the document
+    - After...()
+    - Append...()
+    - Before...()
+    - Clone()
+    - Empty()
+    - Prepend...()
+    - Remove...()
+    - ReplaceWith...()
+    - Unwrap()
+    - Wrap...()
+    - WrapAll...()
+    - WrapInner...()
+
+* property.go : methods that inspect and get the node's properties values.
+    - Attr*(), RemoveAttr(), SetAttr()
+    - AddClass(), HasClass(), RemoveClass(), ToggleClass()
+    - Html()
+    - Length()
+    - Size(), which is an alias for Length()
+    - Text()
+
+* query.go : methods that query, or reflect, a node's identity.
+    - Contains()
+    - Is...()
+
+* traversal.go : methods to traverse the HTML document tree.
+    - Children...()
+    - Contents()
+    - Find...()
+    - Next...()
+    - Parent[s]...()
+    - Prev...()
+    - Siblings...()
+
+* type.go : definition of the types exposed by goquery.
+    - Document
+    - Selection
+    - Matcher
+
+* utilities.go : definition of helper functions (and not methods on a *Selection)
+that are not part of jQuery, but are useful to goquery.
+    - NodeName
+    - OuterHtml
+*/
+package goquery
--- a/vendor/github.com/PuerkitoBio/goquery/expand.go
+++ b/vendor/github.com/PuerkitoBio/goquery/expand.go
@@ -0,0 +1,70 @@
+package goquery
+
+import "golang.org/x/net/html"
+
+// Add adds the selector string's matching nodes to those in the current
+// selection and returns a new Selection object.
+// The selector string is run in the context of the document of the current
+// Selection object.
+func (s *Selection) Add(selector string) *Selection {
+	return s.AddNodes(findWithMatcher([]*html.Node{s.document.rootNode}, compileMatcher(selector))...)
+}
+
+// AddMatcher adds the matcher's matching nodes to those in the current
+// selection and returns a new Selection object.
+// The matcher is run in the context of the document of the current
+// Selection object.
+func (s *Selection) AddMatcher(m Matcher) *Selection {
+	return s.AddNodes(findWithMatcher([]*html.Node{s.document.rootNode}, m)...)
+}
+
+// AddSelection adds the specified Selection object's nodes to those in the
+// current selection and returns a new Selection object.
+func (s *Selection) AddSelection(sel *Selection) *Selection {
+	if sel == nil {
+		return s.AddNodes()
+	}
+	return s.AddNodes(sel.Nodes...)
+}
+
+// Union is an alias for AddSelection.
+func (s *Selection) Union(sel *Selection) *Selection {
+	return s.AddSelection(sel)
+}
+
+// AddNodes adds the specified nodes to those in the
+// current selection and returns a new Selection object.
+func (s *Selection) AddNodes(nodes ...*html.Node) *Selection {
+	return pushStack(s, appendWithoutDuplicates(s.Nodes, nodes, nil))
+}
+
+// AndSelf adds the previous set of elements on the stack to the current set.
+// It returns a new Selection object containing the current Selection combined
+// with the previous one.
+// Deprecated: This function has been deprecated and is now an alias for AddBack().
+func (s *Selection) AndSelf() *Selection {
+	return s.AddBack()
+}
+
+// AddBack adds the previous set of elements on the stack to the current set.
+// It returns a new Selection object containing the current Selection combined
+// with the previous one.
+func (s *Selection) AddBack() *Selection {
+	return s.AddSelection(s.prevSel)
+}
+
+// AddBackFiltered reduces the previous set of elements on the stack to those that
+// match the selector string, and adds them to the current set.
+// It returns a new Selection object containing the current Selection combined
+// with the filtered previous one
+func (s *Selection) AddBackFiltered(selector string) *Selection {
+	return s.AddSelection(s.prevSel.Filter(selector))
+}
+
+// AddBackMatcher reduces the previous set of elements on the stack to those that match
+// the mateher, and adds them to the curernt set.
+// It returns a new Selection object containing the current Selection combined
+// with the filtered previous one
+func (s *Selection) AddBackMatcher(m Matcher) *Selection {
+	return s.AddSelection(s.prevSel.FilterMatcher(m))
+}
--- a/vendor/github.com/PuerkitoBio/goquery/filter.go
+++ b/vendor/github.com/PuerkitoBio/goquery/filter.go
@@ -0,0 +1,163 @@
+package goquery
+
+import "golang.org/x/net/html"
+
+// Filter reduces the set of matched elements to those that match the selector string.
+// It returns a new Selection object for this subset of matching elements.
+func (s *Selection) Filter(selector string) *Selection {
+	return s.FilterMatcher(compileMatcher(selector))
+}
+
+// FilterMatcher reduces the set of matched elements to those that match
+// the given matcher. It returns a new Selection object for this subset
+// of matching elements.
+func (s *Selection) FilterMatcher(m Matcher) *Selection {
+	return pushStack(s, winnow(s, m, true))
+}
+
+// Not removes elements from the Selection that match the selector string.
+// It returns a new Selection object with the matching elements removed.
+func (s *Selection) Not(selector string) *Selection {
+	return s.NotMatcher(compileMatcher(selector))
+}
+
+// NotMatcher removes elements from the Selection that match the given matcher.
+// It returns a new Selection object with the matching elements removed.
+func (s *Selection) NotMatcher(m Matcher) *Selection {
+	return pushStack(s, winnow(s, m, false))
+}
+
+// FilterFunction reduces the set of matched elements to those that pass the function's test.
+// It returns a new Selection object for this subset of elements.
+func (s *Selection) FilterFunction(f func(int, *Selection) bool) *Selection {
+	return pushStack(s, winnowFunction(s, f, true))
+}
+
+// NotFunction removes elements from the Selection that pass the function's test.
+// It returns a new Selection object with the matching elements removed.
+func (s *Selection) NotFunction(f func(int, *Selection) bool) *Selection {
+	return pushStack(s, winnowFunction(s, f, false))
+}
+
+// FilterNodes reduces the set of matched elements to those that match the specified nodes.
+// It returns a new Selection object for this subset of elements.
+func (s *Selection) FilterNodes(nodes ...*html.Node) *Selection {
+	return pushStack(s, winnowNodes(s, nodes, true))
+}
+
+// NotNodes removes elements from the Selection that match the specified nodes.
+// It returns a new Selection object with the matching elements removed.
+func (s *Selection) NotNodes(nodes ...*html.Node) *Selection {
+	return pushStack(s, winnowNodes(s, nodes, false))
+}
+
+// FilterSelection reduces the set of matched elements to those that match a
+// node in the specified Selection object.
+// It returns a new Selection object for this subset of elements.
+func (s *Selection) FilterSelection(sel *Selection) *Selection {
+	if sel == nil {
+		return pushStack(s, winnowNodes(s, nil, true))
+	}
+	return pushStack(s, winnowNodes(s, sel.Nodes, true))
+}
+
+// NotSelection removes elements from the Selection that match a node in the specified
+// Selection object. It returns a new Selection object with the matching elements removed.
+func (s *Selection) NotSelection(sel *Selection) *Selection {
+	if sel == nil {
+		return pushStack(s, winnowNodes(s, nil, false))
+	}
+	return pushStack(s, winnowNodes(s, sel.Nodes, false))
+}
+
+// Intersection is an alias for FilterSelection.
+func (s *Selection) Intersection(sel *Selection) *Selection {
+	return s.FilterSelection(sel)
+}
+
+// Has reduces the set of matched elements to those that have a descendant
+// that matches the selector.
+// It returns a new Selection object with the matching elements.
+func (s *Selection) Has(selector string) *Selection {
+	return s.HasSelection(s.document.Find(selector))
+}
+
+// HasMatcher reduces the set of matched elements to those that have a descendant
+// that matches the matcher.
+// It returns a new Selection object with the matching elements.
+func (s *Selection) HasMatcher(m Matcher) *Selection {
+	return s.HasSelection(s.document.FindMatcher(m))
+}
+
+// HasNodes reduces the set of matched elements to those that have a
+// descendant that matches one of the nodes.
+// It returns a new Selection object with the matching elements.
+func (s *Selection) HasNodes(nodes ...*html.Node) *Selection {
+	return s.FilterFunction(func(_ int, sel *Selection) bool {
+		// Add all nodes that contain one of the specified nodes
+		for _, n := range nodes {
+			if sel.Contains(n) {
+				return true
+			}
+		}
+		return false
+	})
+}
+
+// HasSelection reduces the set of matched elements to those that have a
+// descendant that matches one of the nodes of the specified Selection object.
+// It returns a new Selection object with the matching elements.
+func (s *Selection) HasSelection(sel *Selection) *Selection {
+	if sel == nil {
+		return s.HasNodes()
+	}
+	return s.HasNodes(sel.Nodes...)
+}
+
+// End ends the most recent filtering operation in the current chain and
+// returns the set of matched elements to its previous state.
+func (s *Selection) End() *Selection {
+	if s.prevSel != nil {
+		return s.prevSel
+	}
+	return newEmptySelection(s.document)
+}
+
+// Filter based on the matcher, and the indicator to keep (Filter) or
+// to get rid of (Not) the matching elements.
+func winnow(sel *Selection, m Matcher, keep bool) []*html.Node {
+	// Optimize if keep is requested
+	if keep {
+		return m.Filter(sel.Nodes)
+	}
+	// Use grep
+	return grep(sel, func(i int, s *Selection) bool {
+		return !m.Match(s.Get(0))
+	})
+}
+
+// Filter based on an array of nodes, and the indicator to keep (Filter) or
+// to get rid of (Not) the matching elements.
+func winnowNodes(sel *Selection, nodes []*html.Node, keep bool) []*html.Node {
+	if len(nodes)+len(sel.Nodes) < minNodesForSet {
+		return grep(sel, func(i int, s *Selection) bool {
+			return isInSlice(nodes, s.Get(0)) == keep
+		})
+	}
+
+	set := make(map[*html.Node]bool)
+	for _, n := range nodes {
+		set[n] = true
+	}
+	return grep(sel, func(i int, s *Selection) bool {
+		return set[s.Get(0)] == keep
+	})
+}
+
+// Filter based on a function test, and the indicator to keep (Filter) or
+// to get rid of (Not) the matching elements.
+func winnowFunction(sel *Selection, f func(int, *Selection) bool, keep bool) []*html.Node {
+	return grep(sel, func(i int, s *Selection) bool {
+		return f(i, s) == keep
+	})
+}
--- a/vendor/github.com/PuerkitoBio/goquery/go.mod
+++ b/vendor/github.com/PuerkitoBio/goquery/go.mod
@@ -0,0 +1,8 @@
+module github.com/PuerkitoBio/goquery
+
+require (
+	github.com/andybalholm/cascadia v1.1.0
+	golang.org/x/net v0.0.0-20200202094626-16171245cfb2
+)
+
+go 1.13
--- a/vendor/github.com/PuerkitoBio/goquery/go.sum
+++ b/vendor/github.com/PuerkitoBio/goquery/go.sum
@@ -0,0 +1,8 @@
+github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
+github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
--- a/vendor/github.com/PuerkitoBio/goquery/iteration.go
+++ b/vendor/github.com/PuerkitoBio/goquery/iteration.go
@@ -0,0 +1,39 @@
+package goquery
+
+// Each iterates over a Selection object, executing a function for each
+// matched element. It returns the current Selection object. The function
+// f is called for each element in the selection with the index of the
+// element in that selection starting at 0, and a *Selection that contains
+// only that element.
+func (s *Selection) Each(f func(int, *Selection)) *Selection {
+	for i, n := range s.Nodes {
+		f(i, newSingleSelection(n, s.document))
+	}
+	return s
+}
+
+// EachWithBreak iterates over a Selection object, executing a function for each
+// matched element. It is identical to Each except that it is possible to break
+// out of the loop by returning false in the callback function. It returns the
+// current Selection object.
+func (s *Selection) EachWithBreak(f func(int, *Selection) bool) *Selection {
+	for i, n := range s.Nodes {
+		if !f(i, newSingleSelection(n, s.document)) {
+			return s
+		}
+	}
+	return s
+}
+
+// Map passes each element in the current matched set through a function,
+// producing a slice of string holding the returned values. The function
+// f is called for each element in the selection with the index of the
+// element in that selection starting at 0, and a *Selection that contains
+// only that element.
+func (s *Selection) Map(f func(int, *Selection) string) (result []string) {
+	for i, n := range s.Nodes {
+		result = append(result, f(i, newSingleSelection(n, s.document)))
+	}
+
+	return result
+}
--- a/vendor/github.com/PuerkitoBio/goquery/manipulation.go
+++ b/vendor/github.com/PuerkitoBio/goquery/manipulation.go
@@ -0,0 +1,574 @@
+package goquery
+
+import (
+	"strings"
+
+	"golang.org/x/net/html"
+)
+
+// After applies the selector from the root document and inserts the matched elements
+// after the elements in the set of matched elements.
+//
+// If one of the matched elements in the selection is not currently in the
+// document, it's impossible to insert nodes after it, so it will be ignored.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) After(selector string) *Selection {
+	return s.AfterMatcher(compileMatcher(selector))
+}
+
+// AfterMatcher applies the matcher from the root document and inserts the matched elements
+// after the elements in the set of matched elements.
+//
+// If one of the matched elements in the selection is not currently in the
+// document, it's impossible to insert nodes after it, so it will be ignored.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) AfterMatcher(m Matcher) *Selection {
+	return s.AfterNodes(m.MatchAll(s.document.rootNode)...)
+}
+
+// AfterSelection inserts the elements in the selection after each element in the set of matched
+// elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) AfterSelection(sel *Selection) *Selection {
+	return s.AfterNodes(sel.Nodes...)
+}
+
+// AfterHtml parses the html and inserts it after the set of matched elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) AfterHtml(html string) *Selection {
+	return s.AfterNodes(parseHtml(html)...)
+}
+
+// AfterNodes inserts the nodes after each element in the set of matched elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) AfterNodes(ns ...*html.Node) *Selection {
+	return s.manipulateNodes(ns, true, func(sn *html.Node, n *html.Node) {
+		if sn.Parent != nil {
+			sn.Parent.InsertBefore(n, sn.NextSibling)
+		}
+	})
+}
+
+// Append appends the elements specified by the selector to the end of each element
+// in the set of matched elements, following those rules:
+//
+// 1) The selector is applied to the root document.
+//
+// 2) Elements that are part of the document will be moved to the new location.
+//
+// 3) If there are multiple locations to append to, cloned nodes will be
+// appended to all target locations except the last one, which will be moved
+// as noted in (2).
+func (s *Selection) Append(selector string) *Selection {
+	return s.AppendMatcher(compileMatcher(selector))
+}
+
+// AppendMatcher appends the elements specified by the matcher to the end of each element
+// in the set of matched elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) AppendMatcher(m Matcher) *Selection {
+	return s.AppendNodes(m.MatchAll(s.document.rootNode)...)
+}
+
+// AppendSelection appends the elements in the selection to the end of each element
+// in the set of matched elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) AppendSelection(sel *Selection) *Selection {
+	return s.AppendNodes(sel.Nodes...)
+}
+
+// AppendHtml parses the html and appends it to the set of matched elements.
+func (s *Selection) AppendHtml(html string) *Selection {
+	return s.AppendNodes(parseHtml(html)...)
+}
+
+// AppendNodes appends the specified nodes to each node in the set of matched elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) AppendNodes(ns ...*html.Node) *Selection {
+	return s.manipulateNodes(ns, false, func(sn *html.Node, n *html.Node) {
+		sn.AppendChild(n)
+	})
+}
+
+// Before inserts the matched elements before each element in the set of matched elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) Before(selector string) *Selection {
+	return s.BeforeMatcher(compileMatcher(selector))
+}
+
+// BeforeMatcher inserts the matched elements before each element in the set of matched elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) BeforeMatcher(m Matcher) *Selection {
+	return s.BeforeNodes(m.MatchAll(s.document.rootNode)...)
+}
+
+// BeforeSelection inserts the elements in the selection before each element in the set of matched
+// elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) BeforeSelection(sel *Selection) *Selection {
+	return s.BeforeNodes(sel.Nodes...)
+}
+
+// BeforeHtml parses the html and inserts it before the set of matched elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) BeforeHtml(html string) *Selection {
+	return s.BeforeNodes(parseHtml(html)...)
+}
+
+// BeforeNodes inserts the nodes before each element in the set of matched elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) BeforeNodes(ns ...*html.Node) *Selection {
+	return s.manipulateNodes(ns, false, func(sn *html.Node, n *html.Node) {
+		if sn.Parent != nil {
+			sn.Parent.InsertBefore(n, sn)
+		}
+	})
+}
+
+// Clone creates a deep copy of the set of matched nodes. The new nodes will not be
+// attached to the document.
+func (s *Selection) Clone() *Selection {
+	ns := newEmptySelection(s.document)
+	ns.Nodes = cloneNodes(s.Nodes)
+	return ns
+}
+
+// Empty removes all children nodes from the set of matched elements.
+// It returns the children nodes in a new Selection.
+func (s *Selection) Empty() *Selection {
+	var nodes []*html.Node
+
+	for _, n := range s.Nodes {
+		for c := n.FirstChild; c != nil; c = n.FirstChild {
+			n.RemoveChild(c)
+			nodes = append(nodes, c)
+		}
+	}
+
+	return pushStack(s, nodes)
+}
+
+// Prepend prepends the elements specified by the selector to each element in
+// the set of matched elements, following the same rules as Append.
+func (s *Selection) Prepend(selector string) *Selection {
+	return s.PrependMatcher(compileMatcher(selector))
+}
+
+// PrependMatcher prepends the elements specified by the matcher to each
+// element in the set of matched elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) PrependMatcher(m Matcher) *Selection {
+	return s.PrependNodes(m.MatchAll(s.document.rootNode)...)
+}
+
+// PrependSelection prepends the elements in the selection to each element in
+// the set of matched elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) PrependSelection(sel *Selection) *Selection {
+	return s.PrependNodes(sel.Nodes...)
+}
+
+// PrependHtml parses the html and prepends it to the set of matched elements.
+func (s *Selection) PrependHtml(html string) *Selection {
+	return s.PrependNodes(parseHtml(html)...)
+}
+
+// PrependNodes prepends the specified nodes to each node in the set of
+// matched elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) PrependNodes(ns ...*html.Node) *Selection {
+	return s.manipulateNodes(ns, true, func(sn *html.Node, n *html.Node) {
+		// sn.FirstChild may be nil, in which case this functions like
+		// sn.AppendChild()
+		sn.InsertBefore(n, sn.FirstChild)
+	})
+}
+
+// Remove removes the set of matched elements from the document.
+// It returns the same selection, now consisting of nodes not in the document.
+func (s *Selection) Remove() *Selection {
+	for _, n := range s.Nodes {
+		if n.Parent != nil {
+			n.Parent.RemoveChild(n)
+		}
+	}
+
+	return s
+}
+
+// RemoveFiltered removes the set of matched elements by selector.
+// It returns the Selection of removed nodes.
+func (s *Selection) RemoveFiltered(selector string) *Selection {
+	return s.RemoveMatcher(compileMatcher(selector))
+}
+
+// RemoveMatcher removes the set of matched elements.
+// It returns the Selection of removed nodes.
+func (s *Selection) RemoveMatcher(m Matcher) *Selection {
+	return s.FilterMatcher(m).Remove()
+}
+
+// ReplaceWith replaces each element in the set of matched elements with the
+// nodes matched by the given selector.
+// It returns the removed elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) ReplaceWith(selector string) *Selection {
+	return s.ReplaceWithMatcher(compileMatcher(selector))
+}
+
+// ReplaceWithMatcher replaces each element in the set of matched elements with
+// the nodes matched by the given Matcher.
+// It returns the removed elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) ReplaceWithMatcher(m Matcher) *Selection {
+	return s.ReplaceWithNodes(m.MatchAll(s.document.rootNode)...)
+}
+
+// ReplaceWithSelection replaces each element in the set of matched elements with
+// the nodes from the given Selection.
+// It returns the removed elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) ReplaceWithSelection(sel *Selection) *Selection {
+	return s.ReplaceWithNodes(sel.Nodes...)
+}
+
+// ReplaceWithHtml replaces each element in the set of matched elements with
+// the parsed HTML.
+// It returns the removed elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) ReplaceWithHtml(html string) *Selection {
+	return s.ReplaceWithNodes(parseHtml(html)...)
+}
+
+// ReplaceWithNodes replaces each element in the set of matched elements with
+// the given nodes.
+// It returns the removed elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) ReplaceWithNodes(ns ...*html.Node) *Selection {
+	s.AfterNodes(ns...)
+	return s.Remove()
+}
+
+// SetHtml sets the html content of each element in the selection to
+// specified html string.
+func (s *Selection) SetHtml(html string) *Selection {
+	return setHtmlNodes(s, parseHtml(html)...)
+}
+
+// SetText sets the content of each element in the selection to specified content.
+// The provided text string is escaped.
+func (s *Selection) SetText(text string) *Selection {
+	return s.SetHtml(html.EscapeString(text))
+}
+
+// Unwrap removes the parents of the set of matched elements, leaving the matched
+// elements (and their siblings, if any) in their place.
+// It returns the original selection.
+func (s *Selection) Unwrap() *Selection {
+	s.Parent().Each(func(i int, ss *Selection) {
+		// For some reason, jquery allows unwrap to remove the <head> element, so
+		// allowing it here too. Same for <html>. Why it allows those elements to
+		// be unwrapped while not allowing body is a mystery to me.
+		if ss.Nodes[0].Data != "body" {
+			ss.ReplaceWithSelection(ss.Contents())
+		}
+	})
+
+	return s
+}
+
+// Wrap wraps each element in the set of matched elements inside the first
+// element matched by the given selector. The matched child is cloned before
+// being inserted into the document.
+//
+// It returns the original set of elements.
+func (s *Selection) Wrap(selector string) *Selection {
+	return s.WrapMatcher(compileMatcher(selector))
+}
+
+// WrapMatcher wraps each element in the set of matched elements inside the
+// first element matched by the given matcher. The matched child is cloned
+// before being inserted into the document.
+//
+// It returns the original set of elements.
+func (s *Selection) WrapMatcher(m Matcher) *Selection {
+	return s.wrapNodes(m.MatchAll(s.document.rootNode)...)
+}
+
+// WrapSelection wraps each element in the set of matched elements inside the
+// first element in the given Selection. The element is cloned before being
+// inserted into the document.
+//
+// It returns the original set of elements.
+func (s *Selection) WrapSelection(sel *Selection) *Selection {
+	return s.wrapNodes(sel.Nodes...)
+}
+
+// WrapHtml wraps each element in the set of matched elements inside the inner-
+// most child of the given HTML.
+//
+// It returns the original set of elements.
+func (s *Selection) WrapHtml(html string) *Selection {
+	return s.wrapNodes(parseHtml(html)...)
+}
+
+// WrapNode wraps each element in the set of matched elements inside the inner-
+// most child of the given node. The given node is copied before being inserted
+// into the document.
+//
+// It returns the original set of elements.
+func (s *Selection) WrapNode(n *html.Node) *Selection {
+	return s.wrapNodes(n)
+}
+
+func (s *Selection) wrapNodes(ns ...*html.Node) *Selection {
+	s.Each(func(i int, ss *Selection) {
+		ss.wrapAllNodes(ns...)
+	})
+
+	return s
+}
+
+// WrapAll wraps a single HTML structure, matched by the given selector, around
+// all elements in the set of matched elements. The matched child is cloned
+// before being inserted into the document.
+//
+// It returns the original set of elements.
+func (s *Selection) WrapAll(selector string) *Selection {
+	return s.WrapAllMatcher(compileMatcher(selector))
+}
+
+// WrapAllMatcher wraps a single HTML structure, matched by the given Matcher,
+// around all elements in the set of matched elements. The matched child is
+// cloned before being inserted into the document.
+//
+// It returns the original set of elements.
+func (s *Selection) WrapAllMatcher(m Matcher) *Selection {
+	return s.wrapAllNodes(m.MatchAll(s.document.rootNode)...)
+}
+
+// WrapAllSelection wraps a single HTML structure, the first node of the given
+// Selection, around all elements in the set of matched elements. The matched
+// child is cloned before being inserted into the document.
+//
+// It returns the original set of elements.
+func (s *Selection) WrapAllSelection(sel *Selection) *Selection {
+	return s.wrapAllNodes(sel.Nodes...)
+}
+
+// WrapAllHtml wraps the given HTML structure around all elements in the set of
+// matched elements. The matched child is cloned before being inserted into the
+// document.
+//
+// It returns the original set of elements.
+func (s *Selection) WrapAllHtml(html string) *Selection {
+	return s.wrapAllNodes(parseHtml(html)...)
+}
+
+func (s *Selection) wrapAllNodes(ns ...*html.Node) *Selection {
+	if len(ns) > 0 {
+		return s.WrapAllNode(ns[0])
+	}
+	return s
+}
+
+// WrapAllNode wraps the given node around the first element in the Selection,
+// making all other nodes in the Selection children of the given node. The node
+// is cloned before being inserted into the document.
+//
+// It returns the original set of elements.
+func (s *Selection) WrapAllNode(n *html.Node) *Selection {
+	if s.Size() == 0 {
+		return s
+	}
+
+	wrap := cloneNode(n)
+
+	first := s.Nodes[0]
+	if first.Parent != nil {
+		first.Parent.InsertBefore(wrap, first)
+		first.Parent.RemoveChild(first)
+	}
+
+	for c := getFirstChildEl(wrap); c != nil; c = getFirstChildEl(wrap) {
+		wrap = c
+	}
+
+	newSingleSelection(wrap, s.document).AppendSelection(s)
+
+	return s
+}
+
+// WrapInner wraps an HTML structure, matched by the given selector, around the
+// content of element in the set of matched elements. The matched child is
+// cloned before being inserted into the document.
+//
+// It returns the original set of elements.
+func (s *Selection) WrapInner(selector string) *Selection {
+	return s.WrapInnerMatcher(compileMatcher(selector))
+}
+
+// WrapInnerMatcher wraps an HTML structure, matched by the given selector,
+// around the content of element in the set of matched elements. The matched
+// child is cloned before being inserted into the document.
+//
+// It returns the original set of elements.
+func (s *Selection) WrapInnerMatcher(m Matcher) *Selection {
+	return s.wrapInnerNodes(m.MatchAll(s.document.rootNode)...)
+}
+
+// WrapInnerSelection wraps an HTML structure, matched by the given selector,
+// around the content of element in the set of matched elements. The matched
+// child is cloned before being inserted into the document.
+//
+// It returns the original set of elements.
+func (s *Selection) WrapInnerSelection(sel *Selection) *Selection {
+	return s.wrapInnerNodes(sel.Nodes...)
+}
+
+// WrapInnerHtml wraps an HTML structure, matched by the given selector, around
+// the content of element in the set of matched elements. The matched child is
+// cloned before being inserted into the document.
+//
+// It returns the original set of elements.
+func (s *Selection) WrapInnerHtml(html string) *Selection {
+	return s.wrapInnerNodes(parseHtml(html)...)
+}
+
+// WrapInnerNode wraps an HTML structure, matched by the given selector, around
+// the content of element in the set of matched elements. The matched child is
+// cloned before being inserted into the document.
+//
+// It returns the original set of elements.
+func (s *Selection) WrapInnerNode(n *html.Node) *Selection {
+	return s.wrapInnerNodes(n)
+}
+
+func (s *Selection) wrapInnerNodes(ns ...*html.Node) *Selection {
+	if len(ns) == 0 {
+		return s
+	}
+
+	s.Each(func(i int, s *Selection) {
+		contents := s.Contents()
+
+		if contents.Size() > 0 {
+			contents.wrapAllNodes(ns...)
+		} else {
+			s.AppendNodes(cloneNode(ns[0]))
+		}
+	})
+
+	return s
+}
+
+func parseHtml(h string) []*html.Node {
+	// Errors are only returned when the io.Reader returns any error besides
+	// EOF, but strings.Reader never will
+	nodes, err := html.ParseFragment(strings.NewReader(h), &html.Node{Type: html.ElementNode})
+	if err != nil {
+		panic("goquery: failed to parse HTML: " + err.Error())
+	}
+	return nodes
+}
+
+func setHtmlNodes(s *Selection, ns ...*html.Node) *Selection {
+	for _, n := range s.Nodes {
+		for c := n.FirstChild; c != nil; c = n.FirstChild {
+			n.RemoveChild(c)
+		}
+		for _, c := range ns {
+			n.AppendChild(cloneNode(c))
+		}
+	}
+	return s
+}
+
+// Get the first child that is an ElementNode
+func getFirstChildEl(n *html.Node) *html.Node {
+	c := n.FirstChild
+	for c != nil && c.Type != html.ElementNode {
+		c = c.NextSibling
+	}
+	return c
+}
+
+// Deep copy a slice of nodes.
+func cloneNodes(ns []*html.Node) []*html.Node {
+	cns := make([]*html.Node, 0, len(ns))
+
+	for _, n := range ns {
+		cns = append(cns, cloneNode(n))
+	}
+
+	return cns
+}
+
+// Deep copy a node. The new node has clones of all the original node's
+// children but none of its parents or siblings.
+func cloneNode(n *html.Node) *html.Node {
+	nn := &html.Node{
+		Type:     n.Type,
+		DataAtom: n.DataAtom,
+		Data:     n.Data,
+		Attr:     make([]html.Attribute, len(n.Attr)),
+	}
+
+	copy(nn.Attr, n.Attr)
+	for c := n.FirstChild; c != nil; c = c.NextSibling {
+		nn.AppendChild(cloneNode(c))
+	}
+
+	return nn
+}
+
+func (s *Selection) manipulateNodes(ns []*html.Node, reverse bool,
+	f func(sn *html.Node, n *html.Node)) *Selection {
+
+	lasti := s.Size() - 1
+
+	// net.Html doesn't provide document fragments for insertion, so to get
+	// things in the correct order with After() and Prepend(), the callback
+	// needs to be called on the reverse of the nodes.
+	if reverse {
+		for i, j := 0, len(ns)-1; i < j; i, j = i+1, j-1 {
+			ns[i], ns[j] = ns[j], ns[i]
+		}
+	}
+
+	for i, sn := range s.Nodes {
+		for _, n := range ns {
+			if i != lasti {
+				f(sn, cloneNode(n))
+			} else {
+				if n.Parent != nil {
+					n.Parent.RemoveChild(n)
+				}
+				f(sn, n)
+			}
+		}
+	}
+
+	return s
+}
--- a/vendor/github.com/PuerkitoBio/goquery/property.go
+++ b/vendor/github.com/PuerkitoBio/goquery/property.go
@@ -0,0 +1,275 @@
+package goquery
+
+import (
+	"bytes"
+	"regexp"
+	"strings"
+
+	"golang.org/x/net/html"
+)
+
+var rxClassTrim = regexp.MustCompile("[\t\r\n]")
+
+// Attr gets the specified attribute's value for the first element in the
+// Selection. To get the value for each element individually, use a looping
+// construct such as Each or Map method.
+func (s *Selection) Attr(attrName string) (val string, exists bool) {
+	if len(s.Nodes) == 0 {
+		return
+	}
+	return getAttributeValue(attrName, s.Nodes[0])
+}
+
+// AttrOr works like Attr but returns default value if attribute is not present.
+func (s *Selection) AttrOr(attrName, defaultValue string) string {
+	if len(s.Nodes) == 0 {
+		return defaultValue
+	}
+
+	val, exists := getAttributeValue(attrName, s.Nodes[0])
+	if !exists {
+		return defaultValue
+	}
+
+	return val
+}
+
+// RemoveAttr removes the named attribute from each element in the set of matched elements.
+func (s *Selection) RemoveAttr(attrName string) *Selection {
+	for _, n := range s.Nodes {
+		removeAttr(n, attrName)
+	}
+
+	return s
+}
+
+// SetAttr sets the given attribute on each element in the set of matched elements.
+func (s *Selection) SetAttr(attrName, val string) *Selection {
+	for _, n := range s.Nodes {
+		attr := getAttributePtr(attrName, n)
+		if attr == nil {
+			n.Attr = append(n.Attr, html.Attribute{Key: attrName, Val: val})
+		} else {
+			attr.Val = val
+		}
+	}
+
+	return s
+}
+
+// Text gets the combined text contents of each element in the set of matched
+// elements, including their descendants.
+func (s *Selection) Text() string {
+	var buf bytes.Buffer
+
+	// Slightly optimized vs calling Each: no single selection object created
+	var f func(*html.Node)
+	f = func(n *html.Node) {
+		if n.Type == html.TextNode {
+			// Keep newlines and spaces, like jQuery
+			buf.WriteString(n.Data)
+		}
+		if n.FirstChild != nil {
+			for c := n.FirstChild; c != nil; c = c.NextSibling {
+				f(c)
+			}
+		}
+	}
+	for _, n := range s.Nodes {
+		f(n)
+	}
+
+	return buf.String()
+}
+
+// Size is an alias for Length.
+func (s *Selection) Size() int {
+	return s.Length()
+}
+
+// Length returns the number of elements in the Selection object.
+func (s *Selection) Length() int {
+	return len(s.Nodes)
+}
+
+// Html gets the HTML contents of the first element in the set of matched
+// elements. It includes text and comment nodes.
+func (s *Selection) Html() (ret string, e error) {
+	// Since there is no .innerHtml, the HTML content must be re-created from
+	// the nodes using html.Render.
+	var buf bytes.Buffer
+
+	if len(s.Nodes) > 0 {
+		for c := s.Nodes[0].FirstChild; c != nil; c = c.NextSibling {
+			e = html.Render(&buf, c)
+			if e != nil {
+				return
+			}
+		}
+		ret = buf.String()
+	}
+
+	return
+}
+
+// AddClass adds the given class(es) to each element in the set of matched elements.
+// Multiple class names can be specified, separated by a space or via multiple arguments.
+func (s *Selection) AddClass(class ...string) *Selection {
+	classStr := strings.TrimSpace(strings.Join(class, " "))
+
+	if classStr == "" {
+		return s
+	}
+
+	tcls := getClassesSlice(classStr)
+	for _, n := range s.Nodes {
+		curClasses, attr := getClassesAndAttr(n, true)
+		for _, newClass := range tcls {
+			if !strings.Contains(curClasses, " "+newClass+" ") {
+				curClasses += newClass + " "
+			}
+		}
+
+		setClasses(n, attr, curClasses)
+	}
+
+	return s
+}
+
+// HasClass determines whether any of the matched elements are assigned the
+// given class.
+func (s *Selection) HasClass(class string) bool {
+	class = " " + class + " "
+	for _, n := range s.Nodes {
+		classes, _ := getClassesAndAttr(n, false)
+		if strings.Contains(classes, class) {
+			return true
+		}
+	}
+	return false
+}
+
+// RemoveClass removes the given class(es) from each element in the set of matched elements.
+// Multiple class names can be specified, separated by a space or via multiple arguments.
+// If no class name is provided, all classes are removed.
+func (s *Selection) RemoveClass(class ...string) *Selection {
+	var rclasses []string
+
+	classStr := strings.TrimSpace(strings.Join(class, " "))
+	remove := classStr == ""
+
+	if !remove {
+		rclasses = getClassesSlice(classStr)
+	}
+
+	for _, n := range s.Nodes {
+		if remove {
+			removeAttr(n, "class")
+		} else {
+			classes, attr := getClassesAndAttr(n, true)
+			for _, rcl := range rclasses {
+				classes = strings.Replace(classes, " "+rcl+" ", " ", -1)
+			}
+
+			setClasses(n, attr, classes)
+		}
+	}
+
+	return s
+}
+
+// ToggleClass adds or removes the given class(es) for each element in the set of matched elements.
+// Multiple class names can be specified, separated by a space or via multiple arguments.
+func (s *Selection) ToggleClass(class ...string) *Selection {
+	classStr := strings.TrimSpace(strings.Join(class, " "))
+
+	if classStr == "" {
+		return s
+	}
+
+	tcls := getClassesSlice(classStr)
+
+	for _, n := range s.Nodes {
+		classes, attr := getClassesAndAttr(n, true)
+		for _, tcl := range tcls {
+			if strings.Contains(classes, " "+tcl+" ") {
+				classes = strings.Replace(classes, " "+tcl+" ", " ", -1)
+			} else {
+				classes += tcl + " "
+			}
+		}
+
+		setClasses(n, attr, classes)
+	}
+
+	return s
+}
+
+func getAttributePtr(attrName string, n *html.Node) *html.Attribute {
+	if n == nil {
+		return nil
+	}
+
+	for i, a := range n.Attr {
+		if a.Key == attrName {
+			return &n.Attr[i]
+		}
+	}
+	return nil
+}
+
+// Private function to get the specified attribute's value from a node.
+func getAttributeValue(attrName string, n *html.Node) (val string, exists bool) {
+	if a := getAttributePtr(attrName, n); a != nil {
+		val = a.Val
+		exists = true
+	}
+	return
+}
+
+// Get and normalize the "class" attribute from the node.
+func getClassesAndAttr(n *html.Node, create bool) (classes string, attr *html.Attribute) {
+	// Applies only to element nodes
+	if n.Type == html.ElementNode {
+		attr = getAttributePtr("class", n)
+		if attr == nil && create {
+			n.Attr = append(n.Attr, html.Attribute{
+				Key: "class",
+				Val: "",
+			})
+			attr = &n.Attr[len(n.Attr)-1]
+		}
+	}
+
+	if attr == nil {
+		classes = " "
+	} else {
+		classes = rxClassTrim.ReplaceAllString(" "+attr.Val+" ", " ")
+	}
+
+	return
+}
+
+func getClassesSlice(classes string) []string {
+	return strings.Split(rxClassTrim.ReplaceAllString(" "+classes+" ", " "), " ")
+}
+
+func removeAttr(n *html.Node, attrName string) {
+	for i, a := range n.Attr {
+		if a.Key == attrName {
+			n.Attr[i], n.Attr[len(n.Attr)-1], n.Attr =
+				n.Attr[len(n.Attr)-1], html.Attribute{}, n.Attr[:len(n.Attr)-1]
+			return
+		}
+	}
+}
+
+func setClasses(n *html.Node, attr *html.Attribute, classes string) {
+	classes = strings.TrimSpace(classes)
+	if classes == "" {
+		removeAttr(n, "class")
+		return
+	}
+
+	attr.Val = classes
+}
--- a/vendor/github.com/PuerkitoBio/goquery/query.go
+++ b/vendor/github.com/PuerkitoBio/goquery/query.go
@@ -0,0 +1,49 @@
+package goquery
+
+import "golang.org/x/net/html"
+
+// Is checks the current matched set of elements against a selector and
+// returns true if at least one of these elements matches.
+func (s *Selection) Is(selector string) bool {
+	return s.IsMatcher(compileMatcher(selector))
+}
+
+// IsMatcher checks the current matched set of elements against a matcher and
+// returns true if at least one of these elements matches.
+func (s *Selection) IsMatcher(m Matcher) bool {
+	if len(s.Nodes) > 0 {
+		if len(s.Nodes) == 1 {
+			return m.Match(s.Nodes[0])
+		}
+		return len(m.Filter(s.Nodes)) > 0
+	}
+
+	return false
+}
+
+// IsFunction checks the current matched set of elements against a predicate and
+// returns true if at least one of these elements matches.
+func (s *Selection) IsFunction(f func(int, *Selection) bool) bool {
+	return s.FilterFunction(f).Length() > 0
+}
+
+// IsSelection checks the current matched set of elements against a Selection object
+// and returns true if at least one of these elements matches.
+func (s *Selection) IsSelection(sel *Selection) bool {
+	return s.FilterSelection(sel).Length() > 0
+}
+
+// IsNodes checks the current matched set of elements against the specified nodes
+// and returns true if at least one of these elements matches.
+func (s *Selection) IsNodes(nodes ...*html.Node) bool {
+	return s.FilterNodes(nodes...).Length() > 0
+}
+
+// Contains returns true if the specified Node is within,
+// at any depth, one of the nodes in the Selection object.
+// It is NOT inclusive, to behave like jQuery's implementation, and
+// unlike Javascript's .contains, so if the contained
+// node is itself in the selection, it returns false.
+func (s *Selection) Contains(n *html.Node) bool {
+	return sliceContains(s.Nodes, n)
+}
--- a/vendor/github.com/PuerkitoBio/goquery/traversal.go
+++ b/vendor/github.com/PuerkitoBio/goquery/traversal.go
@@ -0,0 +1,698 @@
+package goquery
+
+import "golang.org/x/net/html"
+
+type siblingType int
+
+// Sibling type, used internally when iterating over children at the same
+// level (siblings) to specify which nodes are requested.
+const (
+	siblingPrevUntil siblingType = iota - 3
+	siblingPrevAll
+	siblingPrev
+	siblingAll
+	siblingNext
+	siblingNextAll
+	siblingNextUntil
+	siblingAllIncludingNonElements
+)
+
+// Find gets the descendants of each element in the current set of matched
+// elements, filtered by a selector. It returns a new Selection object
+// containing these matched elements.
+func (s *Selection) Find(selector string) *Selection {
+	return pushStack(s, findWithMatcher(s.Nodes, compileMatcher(selector)))
+}
+
+// FindMatcher gets the descendants of each element in the current set of matched
+// elements, filtered by the matcher. It returns a new Selection object
+// containing these matched elements.
+func (s *Selection) FindMatcher(m Matcher) *Selection {
+	return pushStack(s, findWithMatcher(s.Nodes, m))
+}
+
+// FindSelection gets the descendants of each element in the current
+// Selection, filtered by a Selection. It returns a new Selection object
+// containing these matched elements.
+func (s *Selection) FindSelection(sel *Selection) *Selection {
+	if sel == nil {
+		return pushStack(s, nil)
+	}
+	return s.FindNodes(sel.Nodes...)
+}
+
+// FindNodes gets the descendants of each element in the current
+// Selection, filtered by some nodes. It returns a new Selection object
+// containing these matched elements.
+func (s *Selection) FindNodes(nodes ...*html.Node) *Selection {
+	return pushStack(s, mapNodes(nodes, func(i int, n *html.Node) []*html.Node {
+		if sliceContains(s.Nodes, n) {
+			return []*html.Node{n}
+		}
+		return nil
+	}))
+}
+
+// Contents gets the children of each element in the Selection,
+// including text and comment nodes. It returns a new Selection object
+// containing these elements.
+func (s *Selection) Contents() *Selection {
+	return pushStack(s, getChildrenNodes(s.Nodes, siblingAllIncludingNonElements))
+}
+
+// ContentsFiltered gets the children of each element in the Selection,
+// filtered by the specified selector. It returns a new Selection
+// object containing these elements. Since selectors only act on Element nodes,
+// this function is an alias to ChildrenFiltered unless the selector is empty,
+// in which case it is an alias to Contents.
+func (s *Selection) ContentsFiltered(selector string) *Selection {
+	if selector != "" {
+		return s.ChildrenFiltered(selector)
+	}
+	return s.Contents()
+}
+
+// ContentsMatcher gets the children of each element in the Selection,
+// filtered by the specified matcher. It returns a new Selection
+// object containing these elements. Since matchers only act on Element nodes,
+// this function is an alias to ChildrenMatcher.
+func (s *Selection) ContentsMatcher(m Matcher) *Selection {
+	return s.ChildrenMatcher(m)
+}
+
+// Children gets the child elements of each element in the Selection.
+// It returns a new Selection object containing these elements.
+func (s *Selection) Children() *Selection {
+	return pushStack(s, getChildrenNodes(s.Nodes, siblingAll))
+}
+
+// ChildrenFiltered gets the child elements of each element in the Selection,
+// filtered by the specified selector. It returns a new
+// Selection object containing these elements.
+func (s *Selection) ChildrenFiltered(selector string) *Selection {
+	return filterAndPush(s, getChildrenNodes(s.Nodes, siblingAll), compileMatcher(selector))
+}
+
+// ChildrenMatcher gets the child elements of each element in the Selection,
+// filtered by the specified matcher. It returns a new
+// Selection object containing these elements.
+func (s *Selection) ChildrenMatcher(m Matcher) *Selection {
+	return filterAndPush(s, getChildrenNodes(s.Nodes, siblingAll), m)
+}
+
+// Parent gets the parent of each element in the Selection. It returns a
+// new Selection object containing the matched elements.
+func (s *Selection) Parent() *Selection {
+	return pushStack(s, getParentNodes(s.Nodes))
+}
+
+// ParentFiltered gets the parent of each element in the Selection filtered by a
+// selector. It returns a new Selection object containing the matched elements.
+func (s *Selection) ParentFiltered(selector string) *Selection {
+	return filterAndPush(s, getParentNodes(s.Nodes), compileMatcher(selector))
+}
+
+// ParentMatcher gets the parent of each element in the Selection filtered by a
+// matcher. It returns a new Selection object containing the matched elements.
+func (s *Selection) ParentMatcher(m Matcher) *Selection {
+	return filterAndPush(s, getParentNodes(s.Nodes), m)
+}
+
+// Closest gets the first element that matches the selector by testing the
+// element itself and traversing up through its ancestors in the DOM tree.
+func (s *Selection) Closest(selector string) *Selection {
+	cs := compileMatcher(selector)
+	return s.ClosestMatcher(cs)
+}
+
+// ClosestMatcher gets the first element that matches the matcher by testing the
+// element itself and traversing up through its ancestors in the DOM tree.
+func (s *Selection) ClosestMatcher(m Matcher) *Selection {
+	return pushStack(s, mapNodes(s.Nodes, func(i int, n *html.Node) []*html.Node {
+		// For each node in the selection, test the node itself, then each parent
+		// until a match is found.
+		for ; n != nil; n = n.Parent {
+			if m.Match(n) {
+				return []*html.Node{n}
+			}
+		}
+		return nil
+	}))
+}
+
+// ClosestNodes gets the first element that matches one of the nodes by testing the
+// element itself and traversing up through its ancestors in the DOM tree.
+func (s *Selection) ClosestNodes(nodes ...*html.Node) *Selection {
+	set := make(map[*html.Node]bool)
+	for _, n := range nodes {
+		set[n] = true
+	}
+	return pushStack(s, mapNodes(s.Nodes, func(i int, n *html.Node) []*html.Node {
+		// For each node in the selection, test the node itself, then each parent
+		// until a match is found.
+		for ; n != nil; n = n.Parent {
+			if set[n] {
+				return []*html.Node{n}
+			}
+		}
+		return nil
+	}))
+}
+
+// ClosestSelection gets the first element that matches one of the nodes in the
+// Selection by testing the element itself and traversing up through its ancestors
+// in the DOM tree.
+func (s *Selection) ClosestSelection(sel *Selection) *Selection {
+	if sel == nil {
+		return pushStack(s, nil)
+	}
+	return s.ClosestNodes(sel.Nodes...)
+}
+
+// Parents gets the ancestors of each element in the current Selection. It
+// returns a new Selection object with the matched elements.
+func (s *Selection) Parents() *Selection {
+	return pushStack(s, getParentsNodes(s.Nodes, nil, nil))
+}
+
+// ParentsFiltered gets the ancestors of each element in the current
+// Selection. It returns a new Selection object with the matched elements.
+func (s *Selection) ParentsFiltered(selector string) *Selection {
+	return filterAndPush(s, getParentsNodes(s.Nodes, nil, nil), compileMatcher(selector))
+}
+
+// ParentsMatcher gets the ancestors of each element in the current
+// Selection. It returns a new Selection object with the matched elements.
+func (s *Selection) ParentsMatcher(m Matcher) *Selection {
+	return filterAndPush(s, getParentsNodes(s.Nodes, nil, nil), m)
+}
+
+// ParentsUntil gets the ancestors of each element in the Selection, up to but
+// not including the element matched by the selector. It returns a new Selection
+// object containing the matched elements.
+func (s *Selection) ParentsUntil(selector string) *Selection {
+	return pushStack(s, getParentsNodes(s.Nodes, compileMatcher(selector), nil))
+}
+
+// ParentsUntilMatcher gets the ancestors of each element in the Selection, up to but
+// not including the element matched by the matcher. It returns a new Selection
+// object containing the matched elements.
+func (s *Selection) ParentsUntilMatcher(m Matcher) *Selection {
+	return pushStack(s, getParentsNodes(s.Nodes, m, nil))
+}
+
+// ParentsUntilSelection gets the ancestors of each element in the Selection,
+// up to but not including the elements in the specified Selection. It returns a
+// new Selection object containing the matched elements.
+func (s *Selection) ParentsUntilSelection(sel *Selection) *Selection {
+	if sel == nil {
+		return s.Parents()
+	}
+	return s.ParentsUntilNodes(sel.Nodes...)
+}
+
+// ParentsUntilNodes gets the ancestors of each element in the Selection,
+// up to but not including the specified nodes. It returns a
+// new Selection object containing the matched elements.
+func (s *Selection) ParentsUntilNodes(nodes ...*html.Node) *Selection {
+	return pushStack(s, getParentsNodes(s.Nodes, nil, nodes))
+}
+
+// ParentsFilteredUntil is like ParentsUntil, with the option to filter the
+// results based on a selector string. It returns a new Selection
+// object containing the matched elements.
+func (s *Selection) ParentsFilteredUntil(filterSelector, untilSelector string) *Selection {
+	return filterAndPush(s, getParentsNodes(s.Nodes, compileMatcher(untilSelector), nil), compileMatcher(filterSelector))
+}
+
+// ParentsFilteredUntilMatcher is like ParentsUntilMatcher, with the option to filter the
+// results based on a matcher. It returns a new Selection object containing the matched elements.
+func (s *Selection) ParentsFilteredUntilMatcher(filter, until Matcher) *Selection {
+	return filterAndPush(s, getParentsNodes(s.Nodes, until, nil), filter)
+}
+
+// ParentsFilteredUntilSelection is like ParentsUntilSelection, with the
+// option to filter the results based on a selector string. It returns a new
+// Selection object containing the matched elements.
+func (s *Selection) ParentsFilteredUntilSelection(filterSelector string, sel *Selection) *Selection {
+	return s.ParentsMatcherUntilSelection(compileMatcher(filterSelector), sel)
+}
+
+// ParentsMatcherUntilSelection is like ParentsUntilSelection, with the
+// option to filter the results based on a matcher. It returns a new
+// Selection object containing the matched elements.
+func (s *Selection) ParentsMatcherUntilSelection(filter Matcher, sel *Selection) *Selection {
+	if sel == nil {
+		return s.ParentsMatcher(filter)
+	}
+	return s.ParentsMatcherUntilNodes(filter, sel.Nodes...)
+}
+
+// ParentsFilteredUntilNodes is like ParentsUntilNodes, with the
+// option to filter the results based on a selector string. It returns a new
+// Selection object containing the matched elements.
+func (s *Selection) ParentsFilteredUntilNodes(filterSelector string, nodes ...*html.Node) *Selection {
+	return filterAndPush(s, getParentsNodes(s.Nodes, nil, nodes), compileMatcher(filterSelector))
+}
+
+// ParentsMatcherUntilNodes is like ParentsUntilNodes, with the
+// option to filter the results based on a matcher. It returns a new
+// Selection object containing the matched elements.
+func (s *Selection) ParentsMatcherUntilNodes(filter Matcher, nodes ...*html.Node) *Selection {
+	return filterAndPush(s, getParentsNodes(s.Nodes, nil, nodes), filter)
+}
+
+// Siblings gets the siblings of each element in the Selection. It returns
+// a new Selection object containing the matched elements.
+func (s *Selection) Siblings() *Selection {
+	return pushStack(s, getSiblingNodes(s.Nodes, siblingAll, nil, nil))
+}
+
+// SiblingsFiltered gets the siblings of each element in the Selection
+// filtered by a selector. It returns a new Selection object containing the
+// matched elements.
+func (s *Selection) SiblingsFiltered(selector string) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingAll, nil, nil), compileMatcher(selector))
+}
+
+// SiblingsMatcher gets the siblings of each element in the Selection
+// filtered by a matcher. It returns a new Selection object containing the
+// matched elements.
+func (s *Selection) SiblingsMatcher(m Matcher) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingAll, nil, nil), m)
+}
+
+// Next gets the immediately following sibling of each element in the
+// Selection. It returns a new Selection object containing the matched elements.
+func (s *Selection) Next() *Selection {
+	return pushStack(s, getSiblingNodes(s.Nodes, siblingNext, nil, nil))
+}
+
+// NextFiltered gets the immediately following sibling of each element in the
+// Selection filtered by a selector. It returns a new Selection object
+// containing the matched elements.
+func (s *Selection) NextFiltered(selector string) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingNext, nil, nil), compileMatcher(selector))
+}
+
+// NextMatcher gets the immediately following sibling of each element in the
+// Selection filtered by a matcher. It returns a new Selection object
+// containing the matched elements.
+func (s *Selection) NextMatcher(m Matcher) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingNext, nil, nil), m)
+}
+
+// NextAll gets all the following siblings of each element in the
+// Selection. It returns a new Selection object containing the matched elements.
+func (s *Selection) NextAll() *Selection {
+	return pushStack(s, getSiblingNodes(s.Nodes, siblingNextAll, nil, nil))
+}
+
+// NextAllFiltered gets all the following siblings of each element in the
+// Selection filtered by a selector. It returns a new Selection object
+// containing the matched elements.
+func (s *Selection) NextAllFiltered(selector string) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingNextAll, nil, nil), compileMatcher(selector))
+}
+
+// NextAllMatcher gets all the following siblings of each element in the
+// Selection filtered by a matcher. It returns a new Selection object
+// containing the matched elements.
+func (s *Selection) NextAllMatcher(m Matcher) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingNextAll, nil, nil), m)
+}
+
+// Prev gets the immediately preceding sibling of each element in the
+// Selection. It returns a new Selection object containing the matched elements.
+func (s *Selection) Prev() *Selection {
+	return pushStack(s, getSiblingNodes(s.Nodes, siblingPrev, nil, nil))
+}
+
+// PrevFiltered gets the immediately preceding sibling of each element in the
+// Selection filtered by a selector. It returns a new Selection object
+// containing the matched elements.
+func (s *Selection) PrevFiltered(selector string) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingPrev, nil, nil), compileMatcher(selector))
+}
+
+// PrevMatcher gets the immediately preceding sibling of each element in the
+// Selection filtered by a matcher. It returns a new Selection object
+// containing the matched elements.
+func (s *Selection) PrevMatcher(m Matcher) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingPrev, nil, nil), m)
+}
+
+// PrevAll gets all the preceding siblings of each element in the
+// Selection. It returns a new Selection object containing the matched elements.
+func (s *Selection) PrevAll() *Selection {
+	return pushStack(s, getSiblingNodes(s.Nodes, siblingPrevAll, nil, nil))
+}
+
+// PrevAllFiltered gets all the preceding siblings of each element in the
+// Selection filtered by a selector. It returns a new Selection object
+// containing the matched elements.
+func (s *Selection) PrevAllFiltered(selector string) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingPrevAll, nil, nil), compileMatcher(selector))
+}
+
+// PrevAllMatcher gets all the preceding siblings of each element in the
+// Selection filtered by a matcher. It returns a new Selection object
+// containing the matched elements.
+func (s *Selection) PrevAllMatcher(m Matcher) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingPrevAll, nil, nil), m)
+}
+
+// NextUntil gets all following siblings of each element up to but not
+// including the element matched by the selector. It returns a new Selection
+// object containing the matched elements.
+func (s *Selection) NextUntil(selector string) *Selection {
+	return pushStack(s, getSiblingNodes(s.Nodes, siblingNextUntil,
+		compileMatcher(selector), nil))
+}
+
+// NextUntilMatcher gets all following siblings of each element up to but not
+// including the element matched by the matcher. It returns a new Selection
+// object containing the matched elements.
+func (s *Selection) NextUntilMatcher(m Matcher) *Selection {
+	return pushStack(s, getSiblingNodes(s.Nodes, siblingNextUntil,
+		m, nil))
+}
+
+// NextUntilSelection gets all following siblings of each element up to but not
+// including the element matched by the Selection. It returns a new Selection
+// object containing the matched elements.
+func (s *Selection) NextUntilSelection(sel *Selection) *Selection {
+	if sel == nil {
+		return s.NextAll()
+	}
+	return s.NextUntilNodes(sel.Nodes...)
+}
+
+// NextUntilNodes gets all following siblings of each element up to but not
+// including the element matched by the nodes. It returns a new Selection
+// object containing the matched elements.
+func (s *Selection) NextUntilNodes(nodes ...*html.Node) *Selection {
+	return pushStack(s, getSiblingNodes(s.Nodes, siblingNextUntil,
+		nil, nodes))
+}
+
+// PrevUntil gets all preceding siblings of each element up to but not
+// including the element matched by the selector. It returns a new Selection
+// object containing the matched elements.
+func (s *Selection) PrevUntil(selector string) *Selection {
+	return pushStack(s, getSiblingNodes(s.Nodes, siblingPrevUntil,
+		compileMatcher(selector), nil))
+}
+
+// PrevUntilMatcher gets all preceding siblings of each element up to but not
+// including the element matched by the matcher. It returns a new Selection
+// object containing the matched elements.
+func (s *Selection) PrevUntilMatcher(m Matcher) *Selection {
+	return pushStack(s, getSiblingNodes(s.Nodes, siblingPrevUntil,
+		m, nil))
+}
+
+// PrevUntilSelection gets all preceding siblings of each element up to but not
+// including the element matched by the Selection. It returns a new Selection
+// object containing the matched elements.
+func (s *Selection) PrevUntilSelection(sel *Selection) *Selection {
+	if sel == nil {
+		return s.PrevAll()
+	}
+	return s.PrevUntilNodes(sel.Nodes...)
+}
+
+// PrevUntilNodes gets all preceding siblings of each element up to but not
+// including the element matched by the nodes. It returns a new Selection
+// object containing the matched elements.
+func (s *Selection) PrevUntilNodes(nodes ...*html.Node) *Selection {
+	return pushStack(s, getSiblingNodes(s.Nodes, siblingPrevUntil,
+		nil, nodes))
+}
+
+// NextFilteredUntil is like NextUntil, with the option to filter
+// the results based on a selector string.
+// It returns a new Selection object containing the matched elements.
+func (s *Selection) NextFilteredUntil(filterSelector, untilSelector string) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingNextUntil,
+		compileMatcher(untilSelector), nil), compileMatcher(filterSelector))
+}
+
+// NextFilteredUntilMatcher is like NextUntilMatcher, with the option to filter
+// the results based on a matcher.
+// It returns a new Selection object containing the matched elements.
+func (s *Selection) NextFilteredUntilMatcher(filter, until Matcher) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingNextUntil,
+		until, nil), filter)
+}
+
+// NextFilteredUntilSelection is like NextUntilSelection, with the
+// option to filter the results based on a selector string. It returns a new
+// Selection object containing the matched elements.
+func (s *Selection) NextFilteredUntilSelection(filterSelector string, sel *Selection) *Selection {
+	return s.NextMatcherUntilSelection(compileMatcher(filterSelector), sel)
+}
+
+// NextMatcherUntilSelection is like NextUntilSelection, with the
+// option to filter the results based on a matcher. It returns a new
+// Selection object containing the matched elements.
+func (s *Selection) NextMatcherUntilSelection(filter Matcher, sel *Selection) *Selection {
+	if sel == nil {
+		return s.NextMatcher(filter)
+	}
+	return s.NextMatcherUntilNodes(filter, sel.Nodes...)
+}
+
+// NextFilteredUntilNodes is like NextUntilNodes, with the
+// option to filter the results based on a selector string. It returns a new
+// Selection object containing the matched elements.
+func (s *Selection) NextFilteredUntilNodes(filterSelector string, nodes ...*html.Node) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingNextUntil,
+		nil, nodes), compileMatcher(filterSelector))
+}
+
+// NextMatcherUntilNodes is like NextUntilNodes, with the
+// option to filter the results based on a matcher. It returns a new
+// Selection object containing the matched elements.
+func (s *Selection) NextMatcherUntilNodes(filter Matcher, nodes ...*html.Node) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingNextUntil,
+		nil, nodes), filter)
+}
+
+// PrevFilteredUntil is like PrevUntil, with the option to filter
+// the results based on a selector string.
+// It returns a new Selection object containing the matched elements.
+func (s *Selection) PrevFilteredUntil(filterSelector, untilSelector string) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingPrevUntil,
+		compileMatcher(untilSelector), nil), compileMatcher(filterSelector))
+}
+
+// PrevFilteredUntilMatcher is like PrevUntilMatcher, with the option to filter
+// the results based on a matcher.
+// It returns a new Selection object containing the matched elements.
+func (s *Selection) PrevFilteredUntilMatcher(filter, until Matcher) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingPrevUntil,
+		until, nil), filter)
+}
+
+// PrevFilteredUntilSelection is like PrevUntilSelection, with the
+// option to filter the results based on a selector string. It returns a new
+// Selection object containing the matched elements.
+func (s *Selection) PrevFilteredUntilSelection(filterSelector string, sel *Selection) *Selection {
+	return s.PrevMatcherUntilSelection(compileMatcher(filterSelector), sel)
+}
+
+// PrevMatcherUntilSelection is like PrevUntilSelection, with the
+// option to filter the results based on a matcher. It returns a new
+// Selection object containing the matched elements.
+func (s *Selection) PrevMatcherUntilSelection(filter Matcher, sel *Selection) *Selection {
+	if sel == nil {
+		return s.PrevMatcher(filter)
+	}
+	return s.PrevMatcherUntilNodes(filter, sel.Nodes...)
+}
+
+// PrevFilteredUntilNodes is like PrevUntilNodes, with the
+// option to filter the results based on a selector string. It returns a new
+// Selection object containing the matched elements.
+func (s *Selection) PrevFilteredUntilNodes(filterSelector string, nodes ...*html.Node) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingPrevUntil,
+		nil, nodes), compileMatcher(filterSelector))
+}
+
+// PrevMatcherUntilNodes is like PrevUntilNodes, with the
+// option to filter the results based on a matcher. It returns a new
+// Selection object containing the matched elements.
+func (s *Selection) PrevMatcherUntilNodes(filter Matcher, nodes ...*html.Node) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingPrevUntil,
+		nil, nodes), filter)
+}
+
+// Filter and push filters the nodes based on a matcher, and pushes the results
+// on the stack, with the srcSel as previous selection.
+func filterAndPush(srcSel *Selection, nodes []*html.Node, m Matcher) *Selection {
+	// Create a temporary Selection with the specified nodes to filter using winnow
+	sel := &Selection{nodes, srcSel.document, nil}
+	// Filter based on matcher and push on stack
+	return pushStack(srcSel, winnow(sel, m, true))
+}
+
+// Internal implementation of Find that return raw nodes.
+func findWithMatcher(nodes []*html.Node, m Matcher) []*html.Node {
+	// Map nodes to find the matches within the children of each node
+	return mapNodes(nodes, func(i int, n *html.Node) (result []*html.Node) {
+		// Go down one level, becausejQuery's Find selects only within descendants
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			if c.Type == html.ElementNode {
+				result = append(result, m.MatchAll(c)...)
+			}
+		}
+		return
+	})
+}
+
+// Internal implementation to get all parent nodes, stopping at the specified
+// node (or nil if no stop).
+func getParentsNodes(nodes []*html.Node, stopm Matcher, stopNodes []*html.Node) []*html.Node {
+	return mapNodes(nodes, func(i int, n *html.Node) (result []*html.Node) {
+		for p := n.Parent; p != nil; p = p.Parent {
+			sel := newSingleSelection(p, nil)
+			if stopm != nil {
+				if sel.IsMatcher(stopm) {
+					break
+				}
+			} else if len(stopNodes) > 0 {
+				if sel.IsNodes(stopNodes...) {
+					break
+				}
+			}
+			if p.Type == html.ElementNode {
+				result = append(result, p)
+			}
+		}
+		return
+	})
+}
+
+// Internal implementation of sibling nodes that return a raw slice of matches.
+func getSiblingNodes(nodes []*html.Node, st siblingType, untilm Matcher, untilNodes []*html.Node) []*html.Node {
+	var f func(*html.Node) bool
+
+	// If the requested siblings are ...Until, create the test function to
+	// determine if the until condition is reached (returns true if it is)
+	if st == siblingNextUntil || st == siblingPrevUntil {
+		f = func(n *html.Node) bool {
+			if untilm != nil {
+				// Matcher-based condition
+				sel := newSingleSelection(n, nil)
+				return sel.IsMatcher(untilm)
+			} else if len(untilNodes) > 0 {
+				// Nodes-based condition
+				sel := newSingleSelection(n, nil)
+				return sel.IsNodes(untilNodes...)
+			}
+			return false
+		}
+	}
+
+	return mapNodes(nodes, func(i int, n *html.Node) []*html.Node {
+		return getChildrenWithSiblingType(n.Parent, st, n, f)
+	})
+}
+
+// Gets the children nodes of each node in the specified slice of nodes,
+// based on the sibling type request.
+func getChildrenNodes(nodes []*html.Node, st siblingType) []*html.Node {
+	return mapNodes(nodes, func(i int, n *html.Node) []*html.Node {
+		return getChildrenWithSiblingType(n, st, nil, nil)
+	})
+}
+
+// Gets the children of the specified parent, based on the requested sibling
+// type, skipping a specified node if required.
+func getChildrenWithSiblingType(parent *html.Node, st siblingType, skipNode *html.Node,
+	untilFunc func(*html.Node) bool) (result []*html.Node) {
+
+	// Create the iterator function
+	var iter = func(cur *html.Node) (ret *html.Node) {
+		// Based on the sibling type requested, iterate the right way
+		for {
+			switch st {
+			case siblingAll, siblingAllIncludingNonElements:
+				if cur == nil {
+					// First iteration, start with first child of parent
+					// Skip node if required
+					if ret = parent.FirstChild; ret == skipNode && skipNode != nil {
+						ret = skipNode.NextSibling
+					}
+				} else {
+					// Skip node if required
+					if ret = cur.NextSibling; ret == skipNode && skipNode != nil {
+						ret = skipNode.NextSibling
+					}
+				}
+			case siblingPrev, siblingPrevAll, siblingPrevUntil:
+				if cur == nil {
+					// Start with previous sibling of the skip node
+					ret = skipNode.PrevSibling
+				} else {
+					ret = cur.PrevSibling
+				}
+			case siblingNext, siblingNextAll, siblingNextUntil:
+				if cur == nil {
+					// Start with next sibling of the skip node
+					ret = skipNode.NextSibling
+				} else {
+					ret = cur.NextSibling
+				}
+			default:
+				panic("Invalid sibling type.")
+			}
+			if ret == nil || ret.Type == html.ElementNode || st == siblingAllIncludingNonElements {
+				return
+			}
+			// Not a valid node, try again from this one
+			cur = ret
+		}
+	}
+
+	for c := iter(nil); c != nil; c = iter(c) {
+		// If this is an ...Until case, test before append (returns true
+		// if the until condition is reached)
+		if st == siblingNextUntil || st == siblingPrevUntil {
+			if untilFunc(c) {
+				return
+			}
+		}
+		result = append(result, c)
+		if st == siblingNext || st == siblingPrev {
+			// Only one node was requested (immediate next or previous), so exit
+			return
+		}
+	}
+	return
+}
+
+// Internal implementation of parent nodes that return a raw slice of Nodes.
+func getParentNodes(nodes []*html.Node) []*html.Node {
+	return mapNodes(nodes, func(i int, n *html.Node) []*html.Node {
+		if n.Parent != nil && n.Parent.Type == html.ElementNode {
+			return []*html.Node{n.Parent}
+		}
+		return nil
+	})
+}
+
+// Internal map function used by many traversing methods. Takes the source nodes
+// to iterate on and the mapping function that returns an array of nodes.
+// Returns an array of nodes mapped by calling the callback function once for
+// each node in the source nodes.
+func mapNodes(nodes []*html.Node, f func(int, *html.Node) []*html.Node) (result []*html.Node) {
+	set := make(map[*html.Node]bool)
+	for i, n := range nodes {
+		if vals := f(i, n); len(vals) > 0 {
+			result = appendWithoutDuplicates(result, vals, set)
+		}
+	}
+	return result
+}
--- a/vendor/github.com/PuerkitoBio/goquery/type.go
+++ b/vendor/github.com/PuerkitoBio/goquery/type.go
@@ -0,0 +1,141 @@
+package goquery
+
+import (
+	"errors"
+	"io"
+	"net/http"
+	"net/url"
+
+	"github.com/andybalholm/cascadia"
+
+	"golang.org/x/net/html"
+)
+
+// Document represents an HTML document to be manipulated. Unlike jQuery, which
+// is loaded as part of a DOM document, and thus acts upon its containing
+// document, GoQuery doesn't know which HTML document to act upon. So it needs
+// to be told, and that's what the Document class is for. It holds the root
+// document node to manipulate, and can make selections on this document.
+type Document struct {
+	*Selection
+	Url      *url.URL
+	rootNode *html.Node
+}
+
+// NewDocumentFromNode is a Document constructor that takes a root html Node
+// as argument.
+func NewDocumentFromNode(root *html.Node) *Document {
+	return newDocument(root, nil)
+}
+
+// NewDocument is a Document constructor that takes a string URL as argument.
+// It loads the specified document, parses it, and stores the root Document
+// node, ready to be manipulated.
+//
+// Deprecated: Use the net/http standard library package to make the request
+// and validate the response before calling goquery.NewDocumentFromReader
+// with the response's body.
+func NewDocument(url string) (*Document, error) {
+	// Load the URL
+	res, e := http.Get(url)
+	if e != nil {
+		return nil, e
+	}
+	return NewDocumentFromResponse(res)
+}
+
+// NewDocumentFromReader returns a Document from an io.Reader.
+// It returns an error as second value if the reader's data cannot be parsed
+// as html. It does not check if the reader is also an io.Closer, the
+// provided reader is never closed by this call. It is the responsibility
+// of the caller to close it if required.
+func NewDocumentFromReader(r io.Reader) (*Document, error) {
+	root, e := html.Parse(r)
+	if e != nil {
+		return nil, e
+	}
+	return newDocument(root, nil), nil
+}
+
+// NewDocumentFromResponse is another Document constructor that takes an http response as argument.
+// It loads the specified response's document, parses it, and stores the root Document
+// node, ready to be manipulated. The response's body is closed on return.
+//
+// Deprecated: Use goquery.NewDocumentFromReader with the response's body.
+func NewDocumentFromResponse(res *http.Response) (*Document, error) {
+	if res == nil {
+		return nil, errors.New("Response is nil")
+	}
+	defer res.Body.Close()
+	if res.Request == nil {
+		return nil, errors.New("Response.Request is nil")
+	}
+
+	// Parse the HTML into nodes
+	root, e := html.Parse(res.Body)
+	if e != nil {
+		return nil, e
+	}
+
+	// Create and fill the document
+	return newDocument(root, res.Request.URL), nil
+}
+
+// CloneDocument creates a deep-clone of a document.
+func CloneDocument(doc *Document) *Document {
+	return newDocument(cloneNode(doc.rootNode), doc.Url)
+}
+
+// Private constructor, make sure all fields are correctly filled.
+func newDocument(root *html.Node, url *url.URL) *Document {
+	// Create and fill the document
+	d := &Document{nil, url, root}
+	d.Selection = newSingleSelection(root, d)
+	return d
+}
+
+// Selection represents a collection of nodes matching some criteria. The
+// initial Selection can be created by using Document.Find, and then
+// manipulated using the jQuery-like chainable syntax and methods.
+type Selection struct {
+	Nodes    []*html.Node
+	document *Document
+	prevSel  *Selection
+}
+
+// Helper constructor to create an empty selection
+func newEmptySelection(doc *Document) *Selection {
+	return &Selection{nil, doc, nil}
+}
+
+// Helper constructor to create a selection of only one node
+func newSingleSelection(node *html.Node, doc *Document) *Selection {
+	return &Selection{[]*html.Node{node}, doc, nil}
+}
+
+// Matcher is an interface that defines the methods to match
+// HTML nodes against a compiled selector string. Cascadia's
+// Selector implements this interface.
+type Matcher interface {
+	Match(*html.Node) bool
+	MatchAll(*html.Node) []*html.Node
+	Filter([]*html.Node) []*html.Node
+}
+
+// compileMatcher compiles the selector string s and returns
+// the corresponding Matcher. If s is an invalid selector string,
+// it returns a Matcher that fails all matches.
+func compileMatcher(s string) Matcher {
+	cs, err := cascadia.Compile(s)
+	if err != nil {
+		return invalidMatcher{}
+	}
+	return cs
+}
+
+// invalidMatcher is a Matcher that always fails to match.
+type invalidMatcher struct{}
+
+func (invalidMatcher) Match(n *html.Node) bool             { return false }
+func (invalidMatcher) MatchAll(n *html.Node) []*html.Node  { return nil }
+func (invalidMatcher) Filter(ns []*html.Node) []*html.Node { return nil }
--- a/vendor/github.com/PuerkitoBio/goquery/utilities.go
+++ b/vendor/github.com/PuerkitoBio/goquery/utilities.go
@@ -0,0 +1,161 @@
+package goquery
+
+import (
+	"bytes"
+
+	"golang.org/x/net/html"
+)
+
+// used to determine if a set (map[*html.Node]bool) should be used
+// instead of iterating over a slice. The set uses more memory and
+// is slower than slice iteration for small N.
+const minNodesForSet = 1000
+
+var nodeNames = []string{
+	html.ErrorNode:    "#error",
+	html.TextNode:     "#text",
+	html.DocumentNode: "#document",
+	html.CommentNode:  "#comment",
+}
+
+// NodeName returns the node name of the first element in the selection.
+// It tries to behave in a similar way as the DOM's nodeName property
+// (https://developer.mozilla.org/en-US/docs/Web/API/Node/nodeName).
+//
+// Go's net/html package defines the following node types, listed with
+// the corresponding returned value from this function:
+//
+//     ErrorNode : #error
+//     TextNode : #text
+//     DocumentNode : #document
+//     ElementNode : the element's tag name
+//     CommentNode : #comment
+//     DoctypeNode : the name of the document type
+//
+func NodeName(s *Selection) string {
+	if s.Length() == 0 {
+		return ""
+	}
+	switch n := s.Get(0); n.Type {
+	case html.ElementNode, html.DoctypeNode:
+		return n.Data
+	default:
+		if n.Type >= 0 && int(n.Type) < len(nodeNames) {
+			return nodeNames[n.Type]
+		}
+		return ""
+	}
+}
+
+// OuterHtml returns the outer HTML rendering of the first item in
+// the selection - that is, the HTML including the first element's
+// tag and attributes.
+//
+// Unlike InnerHtml, this is a function and not a method on the Selection,
+// because this is not a jQuery method (in javascript-land, this is
+// a property provided by the DOM).
+func OuterHtml(s *Selection) (string, error) {
+	var buf bytes.Buffer
+
+	if s.Length() == 0 {
+		return "", nil
+	}
+	n := s.Get(0)
+	if err := html.Render(&buf, n); err != nil {
+		return "", err
+	}
+	return buf.String(), nil
+}
+
+// Loop through all container nodes to search for the target node.
+func sliceContains(container []*html.Node, contained *html.Node) bool {
+	for _, n := range container {
+		if nodeContains(n, contained) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// Checks if the contained node is within the container node.
+func nodeContains(container *html.Node, contained *html.Node) bool {
+	// Check if the parent of the contained node is the container node, traversing
+	// upward until the top is reached, or the container is found.
+	for contained = contained.Parent; contained != nil; contained = contained.Parent {
+		if container == contained {
+			return true
+		}
+	}
+	return false
+}
+
+// Checks if the target node is in the slice of nodes.
+func isInSlice(slice []*html.Node, node *html.Node) bool {
+	return indexInSlice(slice, node) > -1
+}
+
+// Returns the index of the target node in the slice, or -1.
+func indexInSlice(slice []*html.Node, node *html.Node) int {
+	if node != nil {
+		for i, n := range slice {
+			if n == node {
+				return i
+			}
+		}
+	}
+	return -1
+}
+
+// Appends the new nodes to the target slice, making sure no duplicate is added.
+// There is no check to the original state of the target slice, so it may still
+// contain duplicates. The target slice is returned because append() may create
+// a new underlying array. If targetSet is nil, a local set is created with the
+// target if len(target) + len(nodes) is greater than minNodesForSet.
+func appendWithoutDuplicates(target []*html.Node, nodes []*html.Node, targetSet map[*html.Node]bool) []*html.Node {
+	// if there are not that many nodes, don't use the map, faster to just use nested loops
+	// (unless a non-nil targetSet is passed, in which case the caller knows better).
+	if targetSet == nil && len(target)+len(nodes) < minNodesForSet {
+		for _, n := range nodes {
+			if !isInSlice(target, n) {
+				target = append(target, n)
+			}
+		}
+		return target
+	}
+
+	// if a targetSet is passed, then assume it is reliable, otherwise create one
+	// and initialize it with the current target contents.
+	if targetSet == nil {
+		targetSet = make(map[*html.Node]bool, len(target))
+		for _, n := range target {
+			targetSet[n] = true
+		}
+	}
+	for _, n := range nodes {
+		if !targetSet[n] {
+			target = append(target, n)
+			targetSet[n] = true
+		}
+	}
+
+	return target
+}
+
+// Loop through a selection, returning only those nodes that pass the predicate
+// function.
+func grep(sel *Selection, predicate func(i int, s *Selection) bool) (result []*html.Node) {
+	for i, n := range sel.Nodes {
+		if predicate(i, newSingleSelection(n, sel.document)) {
+			result = append(result, n)
+		}
+	}
+	return result
+}
+
+// Creates a new Selection object based on the specified nodes, and keeps the
+// source Selection object on the stack (linked list).
+func pushStack(fromSel *Selection, nodes []*html.Node) *Selection {
+	result := &Selection{nodes, fromSel.document, fromSel}
+	return result
+}
--- a/vendor/github.com/advancedlogic/GoOse/.gitignore
+++ b/vendor/github.com/advancedlogic/GoOse/.gitignore
@@ -0,0 +1,26 @@
+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+*.so
+
+# Folders
+_obj
+_test
+
+# Architecture specific extensions/prefixes
+*.[568vq]
+[568vq].out
+
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+_testmain.go
+
+*.exe
+*.test
+*.prof
+
+target
--- a/vendor/github.com/advancedlogic/GoOse/.travis.yml
+++ b/vendor/github.com/advancedlogic/GoOse/.travis.yml
@@ -0,0 +1,22 @@
+language: go
+
+sudo: false
+
+go:
+  - 1.5
+  - tip
+
+before_install:
+  - go get github.com/mattn/goveralls
+
+before_script:
+  - make deps
+
+script:
+  - make qa
+
+after_failure:
+  - cat ./target/test/report.xml
+
+after_script:
+  - $HOME/gopath/bin/goveralls -covermode=count -coverprofile=target/report/coverage.out -service=travis-ci
--- a/vendor/github.com/advancedlogic/GoOse/Gopkg.lock
+++ b/vendor/github.com/advancedlogic/GoOse/Gopkg.lock
@@ -0,0 +1,129 @@
+# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
+
+
+[[projects]]
+  digest = "1:a62f6ed230a8cd138a9efbe718e7d0b0294f139266f5f55cd942769a9aac8de2"
+  name = "github.com/PuerkitoBio/goquery"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "dc2ec5c7ca4d9aae063b79b9f581dd3ea6afd2b2"
+  version = "v1.4.1"
+
+[[projects]]
+  digest = "1:66b3310cf22cdc96c35ef84ede4f7b9b370971c4025f394c89a2638729653b11"
+  name = "github.com/andybalholm/cascadia"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "901648c87902174f774fac311d7f176f8647bdaa"
+  version = "v1.0.0"
+
+[[projects]]
+  branch = "master"
+  digest = "1:90d593cd34221cf33017a2c944998eb29f5182a15cfbe0cf6e2a0f62cb8bbf29"
+  name = "github.com/araddon/dateparse"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "cfd92a431d0efe36a1b81ca25d15b98aae4dbdb6"
+
+[[projects]]
+  digest = "1:5f1d44e13c151d777a80b0d8ca850347530773e73ccce5995a520cdc6947c205"
+  name = "github.com/fatih/set"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "2c768e3c5489976167bfc42b5c7c92ca783f4389"
+  version = "v0.2.1"
+
+[[projects]]
+  branch = "master"
+  digest = "1:131e93e8b8f318c52b1666e5a8f3af7d236af62f5a477a11973e46af59bca7da"
+  name = "github.com/gigawattio/window"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "0f5467e35573c4ffebddd992935e1f7ceeac0696"
+
+[[projects]]
+  branch = "master"
+  digest = "1:8bacc2118e6cc6a207eb144656537ff7ee0397f8bc8c1f7639519b0de2bb949a"
+  name = "github.com/jaytaylor/html2text"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "57d518f124b0cf46ea2021f25a01396b3522e6fb"
+
+[[projects]]
+  digest = "1:cdb899c199f907ac9fb50495ec71212c95cb5b0e0a8ee0800da0238036091033"
+  name = "github.com/mattn/go-runewidth"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "ce7b0b5c7b45a81508558cd1dba6bb1e4ddb51bb"
+  version = "v0.0.3"
+
+[[projects]]
+  branch = "master"
+  digest = "1:4daa045e1e1f3e23f4b07db6880cdf9f259dab65312dfe244a878e6070faaf77"
+  name = "github.com/olekukonko/tablewriter"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "d4647c9c7a84d847478d890b816b7d8b62b0b279"
+
+[[projects]]
+  branch = "master"
+  digest = "1:362b269942e8508aed33025593090ad4cdbd9077de4b1cd505e6124728e9e856"
+  name = "github.com/ssor/bom"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "6386211fdfcf24c0bfbdaceafd02849ed9a8a509"
+
+[[projects]]
+  branch = "master"
+  digest = "1:1d3f083a0b29a9db5ac57dc4d486dbb97cd58696bae3da441a375916b7e8c53b"
+  name = "golang.org/x/net"
+  packages = [
+    "html",
+    "html/atom",
+    "html/charset",
+  ]
+  pruneopts = "UT"
+  revision = "4bcd98cce591d8c7061bf313d7a3cbad05b58549"
+
+[[projects]]
+  digest = "1:aa4d6967a3237f8367b6bf91503964a77183ecf696f1273e8ad3551bb4412b5f"
+  name = "golang.org/x/text"
+  packages = [
+    "encoding",
+    "encoding/charmap",
+    "encoding/htmlindex",
+    "encoding/internal",
+    "encoding/internal/identifier",
+    "encoding/japanese",
+    "encoding/korean",
+    "encoding/simplifiedchinese",
+    "encoding/traditionalchinese",
+    "encoding/unicode",
+    "internal/gen",
+    "internal/tag",
+    "internal/utf8internal",
+    "language",
+    "runes",
+    "transform",
+    "unicode/cldr",
+  ]
+  pruneopts = "UT"
+  revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
+  version = "v0.3.0"
+
+[solve-meta]
+  analyzer-name = "dep"
+  analyzer-version = 1
+  input-imports = [
+    "github.com/PuerkitoBio/goquery",
+    "github.com/araddon/dateparse",
+    "github.com/fatih/set",
+    "github.com/gigawattio/window",
+    "github.com/jaytaylor/html2text",
+    "golang.org/x/net/html",
+    "golang.org/x/net/html/atom",
+    "golang.org/x/net/html/charset",
+    "golang.org/x/text/transform",
+  ]
+  solver-name = "gps-cdcl"
+  solver-version = 1
--- a/vendor/github.com/advancedlogic/GoOse/Gopkg.toml
+++ b/vendor/github.com/advancedlogic/GoOse/Gopkg.toml
@@ -0,0 +1,58 @@
+# Gopkg.toml example
+#
+# Refer to https://golang.github.io/dep/docs/Gopkg.toml.html
+# for detailed Gopkg.toml documentation.
+#
+# required = ["github.com/user/thing/cmd/thing"]
+# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
+#
+# [[constraint]]
+#   name = "github.com/user/project"
+#   version = "1.0.0"
+#
+# [[constraint]]
+#   name = "github.com/user/project2"
+#   branch = "dev"
+#   source = "github.com/myfork/project2"
+#
+# [[override]]
+#   name = "github.com/x/y"
+#   version = "2.4.0"
+#
+# [prune]
+#   non-go = false
+#   go-tests = true
+#   unused-packages = true
+
+
+[[constraint]]
+  name = "github.com/PuerkitoBio/goquery"
+  version = "1.4.1"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/araddon/dateparse"
+
+[[constraint]]
+  name = "github.com/fatih/set"
+  version = "0.2.1"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/gigawattio/window"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/jaytaylor/html2text"
+
+[[constraint]]
+  branch = "master"
+  name = "golang.org/x/net"
+
+[[constraint]]
+  name = "golang.org/x/text"
+  version = "0.3.0"
+
+[prune]
+  go-tests = true
+  unused-packages = true
--- a/vendor/github.com/advancedlogic/GoOse/LICENSE
+++ b/vendor/github.com/advancedlogic/GoOse/LICENSE
@@ -0,0 +1,202 @@
+Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
--- a/vendor/github.com/advancedlogic/GoOse/Makefile
+++ b/vendor/github.com/advancedlogic/GoOse/Makefile
@@ -0,0 +1,203 @@
+# MAKEFILE
+#
+# @author      Nicola Asuni <info@tecnick.com>
+# @link        https://github.com/advancedlogic/GoOse
+#
+# This file is intended to be executed in a Linux-compatible system.
+# It also assumes that the project has been cloned in the right path under GOPATH:
+# $GOPATH/src/github.com/advancedlogic/GoOse
+#
+# ------------------------------------------------------------------------------
+
+# List special make targets that are not associated with files
+.PHONY: help all test format fmtcheck vet lint coverage cyclo ineffassign misspell structcheck varcheck errcheck gosimple astscan qa deps clean nuke
+
+# Use bash as shell (Note: Ubuntu now uses dash which doesn't support PIPESTATUS).
+SHELL=/bin/bash
+
+# CVS path (path to the parent dir containing the project)
+CVSPATH=github.com/advancedlogic
+
+# Project owner
+OWNER=advancedlogic
+
+# Project vendor
+VENDOR=advancedlogic
+
+# Project name
+PROJECT=GoOse
+
+# Project version
+VERSION=$(shell cat VERSION)
+
+# Name of RPM or DEB package
+PKGNAME=${VENDOR}-${PROJECT}
+
+# Current directory
+CURRENTDIR=$(shell pwd)
+
+# GO lang path
+ifneq ($(GOPATH),)
+	ifeq ($(findstring $(GOPATH),$(CURRENTDIR)),)
+		# the defined GOPATH is not valid
+		GOPATH=
+	endif
+endif
+ifeq ($(GOPATH),)
+	# extract the GOPATH
+	GOPATH=$(firstword $(subst /src/, ,$(CURRENTDIR)))
+endif
+
+# --- MAKE TARGETS ---
+
+# Display general help about this command
+help:
+	@echo ""
+	@echo "$(PROJECT) Makefile."
+	@echo "GOPATH=$(GOPATH)"
+	@echo "The following commands are available:"
+	@echo ""
+	@echo "    make qa          : Run all the tests"
+	@echo "    make test        : Run the unit tests"
+	@echo ""
+	@echo "    make format      : Format the source code"
+	@echo "    make fmtcheck    : Check if the source code has been formatted"
+	@echo "    make vet         : Check for suspicious constructs"
+	@echo "    make lint        : Check for style errors"
+	@echo "    make coverage    : Generate the coverage report"
+	@echo "    make cyclo       : Generate the cyclomatic complexity report"
+	@echo "    make ineffassign : Detect ineffectual assignments"
+	@echo "    make misspell    : Detect commonly misspelled words in source files"
+	@echo "    make structcheck : Find unused struct fields"
+	@echo "    make varcheck    : Find unused global variables and constants"
+	@echo "    make errcheck    : Check that error return values are used"
+	@echo "    make gosimple    : Suggest code simplifications"
+	@echo "    make astscan     : GO AST scanner"
+	@echo ""
+	@echo "    make docs        : Generate source code documentation"
+	@echo ""
+	@echo "    make deps        : Get the dependencies"
+	@echo "    make clean       : Remove any build artifact"
+	@echo "    make nuke        : Deletes any intermediate file"
+	@echo ""
+
+
+# Alias for help target
+all: help
+
+# Run the unit tests
+test:
+	@mkdir -p target/test
+	@mkdir -p target/report
+	GOPATH=$(GOPATH) \
+	go test \
+	-covermode=atomic \
+	-bench=. \
+	-race \
+	-cpuprofile=target/report/cpu.out \
+	-memprofile=target/report/mem.out \
+	-mutexprofile=target/report/mutex.out \
+	-coverprofile=target/report/coverage.out \
+	-v ./... | \
+	tee >(PATH=$(GOPATH)/bin:$(PATH) go-junit-report > target/test/report.xml); \
+	test $${PIPESTATUS[0]} -eq 0
+
+# Format the source code
+format:
+	@find . -type f -name "*.go" -exec gofmt -s -w {} \;
+
+# Check if the source code has been formatted
+fmtcheck:
+	@mkdir -p target
+	@find . -type f -name "*.go" -exec gofmt -s -d {} \; | tee target/format.diff
+	@test ! -s target/format.diff || { echo "ERROR: the source code has not been formatted - please use 'make format' or 'gofmt'"; exit 1; }
+
+# Check for syntax errors
+vet:
+	GOPATH=$(GOPATH) go vet .
+
+# Check for style errors
+lint:
+	GOPATH=$(GOPATH) PATH=$(GOPATH)/bin:$(PATH) golint .
+
+# Generate the coverage report
+coverage:
+	@mkdir -p target/report
+	GOPATH=$(GOPATH) \
+	go tool cover -html=target/report/coverage.out -o target/report/coverage.html
+
+# Report cyclomatic complexity
+cyclo:
+	@mkdir -p target/report
+	GOPATH=$(GOPATH) gocyclo -avg ./ | tee target/report/cyclo.txt ; test $${PIPESTATUS[0]} -eq 0
+
+# Detect ineffectual assignments
+ineffassign:
+	@mkdir -p target/report
+	GOPATH=$(GOPATH) ineffassign ./ | tee target/report/ineffassign.txt ; test $${PIPESTATUS[0]} -eq 0
+
+# Detect commonly misspelled words in source files
+misspell:
+	@mkdir -p target/report
+	GOPATH=$(GOPATH) misspell -error ./*.go | tee target/report/misspell.txt
+
+# Find unused struct fields
+structcheck:
+	@mkdir -p target/report
+	GOPATH=$(GOPATH) structcheck -a ./ | tee target/report/structcheck.txt
+
+# Find unused global variables and constants
+varcheck:
+	@mkdir -p target/report
+	GOPATH=$(GOPATH) varcheck -e ./ | tee target/report/varcheck.txt
+
+# Check that error return values are used
+errcheck:
+	@mkdir -p target/report
+	GOPATH=$(GOPATH) errcheck ./ | tee target/report/errcheck.txt
+
+# Suggest code simplifications
+gosimple:
+	@mkdir -p target/report
+	GOPATH=$(GOPATH) gosimple ./ | tee target/report/gosimple.txt
+
+# AST scanner
+astscan:
+	@mkdir -p target/report
+	GOPATH=$(GOPATH) gas .//*.go | tee target/report/astscan.txt
+
+# Generate source docs
+docs:
+	@mkdir -p target/docs
+	nohup sh -c 'GOPATH=$(GOPATH) godoc -http=127.0.0.1:6060' > target/godoc_server.log 2>&1 &
+	wget --directory-prefix=target/docs/ --execute robots=off --retry-connrefused --recursive --no-parent --adjust-extension --page-requisites --convert-links http://127.0.0.1:6060/pkg/github.com/${VENDOR}/${PROJECT}/ ; kill -9 `lsof -ti :6060`
+	@echo '<html><head><meta http-equiv="refresh" content="0;./127.0.0.1:6060/pkg/'${CVSPATH}'/'${PROJECT}'/index.html"/></head><a href="./127.0.0.1:6060/pkg/'${CVSPATH}'/'${PROJECT}'/index.html">'${PKGNAME}' Documentation ...</a></html>' > target/docs/index.html
+
+# Alias to run all quality-assurance checks
+qa: fmtcheck test vet lint coverage cyclo ineffassign misspell structcheck varcheck errcheck gosimple astscan
+
+# --- INSTALL ---
+
+# Get the dependencies
+deps:
+	GOPATH=$(GOPATH) go get ./...
+	GOPATH=$(GOPATH) go get github.com/golang/lint/golint
+	GOPATH=$(GOPATH) go get github.com/jstemmer/go-junit-report
+	GOPATH=$(GOPATH) go get github.com/axw/gocov/gocov
+	GOPATH=$(GOPATH) go get github.com/fzipp/gocyclo
+	GOPATH=$(GOPATH) go get github.com/gordonklaus/ineffassign
+	GOPATH=$(GOPATH) go get github.com/client9/misspell/cmd/misspell
+	GOPATH=$(GOPATH) go get github.com/opennota/check/cmd/structcheck
+	GOPATH=$(GOPATH) go get github.com/opennota/check/cmd/varcheck
+	GOPATH=$(GOPATH) go get github.com/kisielk/errcheck
+	GOPATH=$(GOPATH) go get honnef.co/go/tools/cmd/gosimple
+	GOPATH=$(GOPATH) go get github.com/GoASTScanner/gas
+
+# Remove any build artifact
+clean:
+	GOPATH=$(GOPATH) go clean ./...
+
+# Deletes any intermediate file
+nuke:
+	rm -rf ./target
+	GOPATH=$(GOPATH) go clean -i ./...
--- a/vendor/github.com/advancedlogic/GoOse/README.md
+++ b/vendor/github.com/advancedlogic/GoOse/README.md
@@ -0,0 +1,86 @@
+# GoOse
+
+*HTML Content / Article Extractor in Golang*
+
+[![Build Status](https://secure.travis-ci.org/advancedlogic/GoOse.png?branch=master)](https://travis-ci.org/advancedlogic/GoOse?branch=master)
+[![Coverage Status](https://coveralls.io/repos/advancedlogic/GoOse/badge.svg?branch=master&service=github)](https://coveralls.io/github/advancedlogic/GoOse?branch=master)
+[![Go Report Card](https://goreportcard.com/badge/github.com/advancedlogic/GoOse)](https://goreportcard.com/report/github.com/advancedlogic/GoOse)
+[![GoDoc](https://godoc.org/github.com/advancedlogic/GoOse?status.svg)](http://godoc.org/github.com/advancedlogic/GoOse)
+
+
+## Description
+
+This is a golang port of "Goose" originaly licensed to Gravity.com
+under one or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.
+
+Golang port was written by Antonio Linari
+
+Gravity.com licenses this file
+to you under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+## INSTALL
+
+```bash
+go get github.com/advancedlogic/GoOse
+```
+
+## HOW TO USE IT
+
+```Go
+package main
+
+import (
+	"github.com/advancedlogic/GoOse"
+)
+
+func main() {
+	g := goose.New()
+	article, _ := g.ExtractFromURL("http://edition.cnn.com/2012/07/08/opinion/banzi-ted-open-source/index.html")
+	println("title", article.Title)
+	println("description", article.MetaDescription)
+	println("keywords", article.MetaKeywords)
+	println("content", article.CleanedText)
+	println("url", article.FinalURL)
+	println("top image", article.TopImage)
+}
+```
+
+## Development - Getting started
+
+This application is written in GO language, please refere to the guides in https://golang.org for getting started.
+
+This project include a Makefile that allows you to test and build the project with simple commands.
+To see all available options:
+```bash
+make help
+```
+
+Before committing the code, please check if it passes all tests using
+```bash
+make deps
+make qa
+```
+
+## TODO
+- [ ] better organize code
+- [ ] improve "xpath" like queries
+- [ ] add other image extractions techniques (imagemagick)
+
+## THANKS TO
+```
+@Martin Angers for goquery
+@Fatih Arslan for set
+GoLang team for the amazing language and net/html
+```
--- a/vendor/github.com/advancedlogic/GoOse/VERSION
+++ b/vendor/github.com/advancedlogic/GoOse/VERSION
@@ -0,0 +1 @@
+1.1.0
--- a/vendor/github.com/advancedlogic/GoOse/article.go
+++ b/vendor/github.com/advancedlogic/GoOse/article.go
@@ -0,0 +1,40 @@
+package goose
+
+import (
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/fatih/set"
+)
+
+// Article is a collection of properties extracted from the HTML body
+type Article struct {
+	Title           string             `json:"title,omitempty"`
+	TitleUnmodified string             `json:"titleunmodified,omitempty"`
+	CleanedText     string             `json:"content,omitempty"`
+	MetaDescription string             `json:"description,omitempty"`
+	MetaLang        string             `json:"lang,omitempty"`
+	MetaFavicon     string             `json:"favicon,omitempty"`
+	MetaKeywords    string             `json:"keywords,omitempty"`
+	CanonicalLink   string             `json:"canonicalurl,omitempty"`
+	Domain          string             `json:"domain,omitempty"`
+	TopNode         *goquery.Selection `json:"-"`
+	TopImage        string             `json:"image,omitempty"`
+	Tags            *set.Set           `json:"tags,omitempty"`
+	Movies          *set.Set           `json:"movies,omitempty"`
+	FinalURL        string             `json:"url,omitempty"`
+	LinkHash        string             `json:"linkhash,omitempty"`
+	RawHTML         string             `json:"rawhtml,omitempty"`
+	Doc             *goquery.Document  `json:"-"`
+	Links           []string           `json:"links,omitempty"`
+	PublishDate     *time.Time         `json:"publishdate,omitempty"`
+	AdditionalData  map[string]string  `json:"additionaldata,omitempty"`
+	Delta           int64              `json:"delta,omitempty"`
+}
+
+// ToString is a simple method to just show the title
+// TODO: add more fields and pretty print
+func (article *Article) ToString() string {
+	out := article.Title
+	return out
+}
--- a/vendor/github.com/advancedlogic/GoOse/charset.go
+++ b/vendor/github.com/advancedlogic/GoOse/charset.go
@@ -0,0 +1,92 @@
+package goose
+
+import (
+	"log"
+	"strings"
+	"unicode/utf8"
+
+	"golang.org/x/net/html/charset"
+	"golang.org/x/text/transform"
+)
+
+// NormaliseCharset Overrides/fixes charset names to something we can parse.
+// Fixes common mispellings and uses a canonical name for equivalent encodings.
+// @see https://encoding.spec.whatwg.org#names-and-labels
+func NormaliseCharset(characterSet string) string {
+	characterSet = strings.ToUpper(characterSet)
+	switch characterSet {
+	case "UTF8", "UT-8", "UTR-8", "UFT-8", "UTF8-WITHOUT-BOM", "UTF8_GENERAL_CI":
+		return "UTF-8"
+	// override Japanese
+	// CP943: IBM OS/2 Japanese, superset of Cp932 and Shift-JIS
+	case "CP943", "CP943C", "SIFT_JIS", "SHIFT-JIS":
+		return "SHIFT_JIS"
+	// override Korean
+	case "EUC-KR", "MS949", "KSC5601", "WINDOWS-949", "KS_C_5601-1987", "KSC_5601":
+		return "UHC"
+	// override Thai
+	//case "TIS-620", "WINDOWS-874":
+	//	return "ISO-8859-11"
+	// override latin-2
+	case "LATIN2_HUNGARIAN_CI", "LATIN2":
+		return "LATIN-2"
+	// override cyrillic
+	case "WIN1251", "WIN-1251", "WINDOWS-1251":
+		return "CP1251"
+	// override Hebrew
+	case "WINDOWS-1255":
+		return "ISO-8859-8"
+	// override Turkish
+	//case "WINDOWS-1254":
+	//	return "ISO-8859-9"
+	// override the parsing of ISO-8859-1 to behave as Windows-1252 (CP1252):
+	// in ISO-8859-1, everything from 128-255 in the ASCII table are ctrl characters,
+	// whilst in CP1252 they're symbols
+	// override Baltic
+	case "WINDOWS-1257":
+		return "ISO-8859-13"
+	case "ANSI", "LATIN-1", "ISO", "RFC", "MACINTOSH", "8859-1", "8859-15", "ISO8859-1", "ISO8859-15", "ISO-8559-1", "ISO-8859-1", "ISO-8859-15":
+		return "CP1252"
+	}
+	return characterSet
+}
+
+// UTF8encode converts a string from the source character set to UTF-8, skipping invalid byte sequences
+// @see http://stackoverflow.com/questions/32512500/ignore-illegal-bytes-when-decoding-text-with-go
+func UTF8encode(raw string, sourceCharset string) string {
+	enc, name := charset.Lookup(sourceCharset)
+	if nil == enc {
+		log.Println("Cannot convert from", sourceCharset, ":", name)
+		return raw
+	}
+
+	dst := make([]byte, len(raw))
+	d := enc.NewDecoder()
+
+	var (
+		in  int
+		out int
+	)
+	for in < len(raw) {
+		// Do the transformation
+		ndst, nsrc, err := d.Transform(dst[out:], []byte(raw[in:]), true)
+		in += nsrc
+		out += ndst
+		if err == nil {
+			// Completed transformation
+			break
+		}
+		if err == transform.ErrShortDst {
+			// Our output buffer is too small, so we need to grow it
+			t := make([]byte, (cap(dst)+1)*2)
+			copy(t, dst)
+			dst = t
+			continue
+		}
+		// We're here because of at least one illegal character. Skip over the current rune
+		// and try again.
+		_, width := utf8.DecodeRuneInString(raw[in:])
+		in += width
+	}
+	return string(dst)
+}
--- a/vendor/github.com/advancedlogic/GoOse/cleaner.go
+++ b/vendor/github.com/advancedlogic/GoOse/cleaner.go
@@ -0,0 +1,568 @@
+package goose
+
+import (
+	"container/list"
+	"github.com/PuerkitoBio/goquery"
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/atom"
+	"log"
+	"regexp"
+	"strings"
+)
+
+var whitelistedTextAtomTypes = []atom.Atom{atom.Span, atom.Em, atom.I, atom.Strong, atom.B, atom.P, atom.H1, atom.H2, atom.H3, atom.H4}
+var whitelistedExtAtomTypes = []atom.Atom{atom.A, atom.Span, atom.Em, atom.I, atom.Strong, atom.B, atom.P, atom.H1, atom.H2, atom.H3, atom.H4}
+
+// Cleaner removes menus, ads, sidebars, etc. and leaves the main content
+type Cleaner struct {
+	config Configuration
+}
+
+// NewCleaner returns a new instance of a Cleaner
+func NewCleaner(config Configuration) Cleaner {
+	return Cleaner{
+		config: config,
+	}
+}
+
+// replaceTagWithContents removes the tag, replacing it with its text contents
+// e.g. "<em>some text</em>" becomes "some text"
+func replaceTagWithContents(tagSelection *goquery.Selection, collapsibleAtomTypes []atom.Atom) {
+	if tagSelection.Length() == 0 {
+		return
+	}
+	node := tagSelection.Get(0)
+	node.Data = tagSelection.Text()
+	node.Type = html.TextNode
+	if node.FirstChild == nil {
+		node.Attr = []html.Attribute{}
+		node.DataAtom = 0
+		node.FirstChild = nil
+		node.LastChild = nil
+	} else {
+		// If all children are text only, the parent already contains the text, so drop them
+		collapseTextNodes(node, collapsibleAtomTypes)
+	}
+}
+
+func isAtomTypeWhitelisted(t atom.Atom, whitelist []atom.Atom) bool {
+	for _, allowed := range whitelist {
+		if t == allowed {
+			return true
+		}
+	}
+	return false
+}
+
+func collapseTextNodes(node *html.Node, collapsibleAtomTypes []atom.Atom) {
+	if node.FirstChild == nil {
+		return
+	}
+
+	if !isAtomTypeWhitelisted(node.DataAtom, collapsibleAtomTypes) {
+		return
+	}
+
+	if node.FirstChild.DataAtom == 0 && node.FirstChild == node.LastChild {
+		// this tag only contains a single textual node, already contained in the parent
+		node.Attr = []html.Attribute{}
+		node.Type = html.TextNode
+		node.DataAtom = 0
+		node.FirstChild = nil
+		node.LastChild = nil
+		return
+	}
+
+	// If all children are text only, the parent already contains the text, so drop them
+	allTextNodes := true
+	for c := node.FirstChild; c != nil; c = c.NextSibling {
+		// attempt collapsing recursively
+		collapseTextNodes(c, collapsibleAtomTypes)
+		if c.DataAtom != 0 {
+			// not collapsed
+			allTextNodes = false
+			break
+		}
+	}
+	if allTextNodes {
+		// text already contained in the parent node => drop children
+		node.Attr = []html.Attribute{}
+		node.Type = html.TextNode
+		node.DataAtom = 0
+		node.FirstChild = nil
+		node.LastChild = nil
+	}
+}
+
+var divToPElementsPattern = regexp.MustCompile("<(a|blockquote|dl|div|img|ol|p|pre|table|ul)")
+var tabsRegEx = regexp.MustCompile(`\t|^\s+$]`)
+var removeVisibilityStyleRegEx = regexp.MustCompile("visibility:[ ]*hidden|display:[ ]*none")
+var removeNodesRegEx = regexp.MustCompile("" +
+	"[Cc]omentario|" +
+	"[Ff]ooter|" +
+	"^fn$|" +
+	"^inset$|" +
+	"^print$|" +
+	"^scroll$|" +
+	"^side$|" +
+	"^side_|" +
+	"^widget$|" +
+	"^ab[0-9]$|" +
+	"[_-]ads$|" +
+	"^ad[s]?[ _-]|" +
+	"[_-]ad[s]?[_-]|" +
+	"^ADX_CLIENTSIDE$|" +
+	"ajoutVideo|" +
+	"^alerts|" +
+	"^Anchor$|" +
+	"articleheadings|" +
+	"_articles|" +
+	"^article-gallery-embedded$|" +
+	"author|" +
+	"author-dropdown|" +
+	"^banner|" +
+	"^bar$|" +
+	"blog-pager|" +
+	"brass\\-rail|" +
+	"breadcrumbs|" +
+	"button|" +
+	"byline|" +
+	"cabecalho|" +
+	"^caption$|" +
+	"carousel|" +
+	"^click|" +
+	"cnnStryHghLght|" +
+	"cnn_html_slideshow|" +
+	"cnn_strycaptiontxt|" +
+	"cnn_strylftcntnt|" +
+	"cnn_stryspcvbx|" +
+	"combx|" +
+	"comment|" +
+	"commercial|" +
+	"communitypromo|" +
+	"^comscore$|" +
+	"contact|" +
+	"contentTools2|" +
+	"controls|" +
+	"cookie|" +
+	"CoversMainContent|" +
+	"^css-|" +
+	"^critical-alerts$|" +
+	"^date$|" +
+	"detail_new_|" +
+	"downloadLink|" +
+	"^DYSRC$|" +
+	"^early-body|" +
+	"ec_blogs|" +
+	"^[^entry-]more.*$|" +
+	"error|" +
+	"[^-]facebook|" +
+	"facebook-broadcasting|" +
+	"^fb-root$|" +
+	"^feed[_-]|" +
+	"figcaption|" +
+	"footnote|" +
+	"foot|" +
+	"footer|" +
+	"^ga-track$|" +
+	" google |" +
+	"^gstl_|" +
+	"^GS-UH$|" +
+	"^guide$|" +
+	"header|" +
+	"hidden|" +
+	"img_popup_single|" +
+	"inline-share-tools|" +
+	"inread|" +
+	"^interstitial-ad-modal$|" +
+	"^Inv[0-9]$|" +
+	"js_replies|" +
+	"[Kk]ona[Ff]ilter|" +
+	"^kxhead$|" +
+	"leading|" +
+	"^lede[_-]container$|" +
+	"legende?|" +
+	"^lightningjs-|" +
+	"links|" +
+	"^login-modal$|" +
+	"^lui-mini-profile-body$|" +
+	"^marginalia|" +
+	"^marketing[_-]|" +
+	"^masthead|" +
+	"mediaarticlerelated|" +
+	"^media[_-]viewer$|" +
+	"menu|" +
+	"menucontainer|" +
+	"meta$|" +
+	"^moat$|" +
+	"moreNews|" +
+	"^Moses$|" +
+	"^nav[_-]|" +
+	"navbar|" +
+	"[Nn]avigation|" +
+	"newsUnder|" +
+	"^oauth|" +
+	"^overlay[_-]wrapper|" +
+	"pagetools|" +
+	"[_-]paid[_-]|" +
+	"panelss2|" +
+	"panesCity|" +
+	"player|" +
+	"PopularQuestions|" +
+	"popup|" +
+	"post[_-]attributes|" +
+	"post[_-]title|" +
+	"preview|" +
+	"[_-]print[_-]|" +
+	"products\\-events|" +
+	"^prop[0-9]$|" +
+	"^pulse-loaders|" +
+	"^rail$|" +
+	"recommend|" +
+	"^registration-modal$|" +
+	"relacionado|" +
+	"related|" +
+	"remote|" +
+	"retweet|" +
+	"^ribbon$|" +
+	"rightBlock|" +
+	"rss|" +
+	"runaroundLeft|" +
+	"search[_-]|" +
+	"share[_-]|" +
+	"shoutbox|" +
+	"sidebar|" +
+	"^simplereach$|" +
+	"^site[_-]index$|" +
+	"site[_-]box|" +
+	"site[_-]nav|" +
+	"skyscraper|" +
+	"social[Nn]etworking|" +
+	"social_|" +
+	"social\\-share|" +
+	"social\\-count|" +
+	"socialtools|" +
+	"source|" +
+	"^speed-bump-wrapper$|" +
+	"[_-]spinner$|" +
+	"^Splash$|" +
+	"sponsor|" +
+	"^spr-|" +
+	"storytopbar\\-bucket|" +
+	"^stream-sidebar|" +
+	"sub_nav|" +
+	"subscribe|" +
+	"subscription|" +
+	"^suggestions$|" +
+	"tabsCity|" +
+	"tag_|" +
+	"tags|" +
+	"teaser|" +
+	"the_answers|" +
+	"timestamp|" +
+	"tools|" +
+	"tooltip|" +
+	"^Top[0-9]?$|" +
+	"^TopAd[0-9]?$|" +
+	"[_-]track[_-]|" +
+	"tracking|" +
+	"[^-]twitter|" +
+	"-uix-button|" +
+	"updateBrowser|" +
+	"^username-modal$|" +
+	"^user-|" +
+	"utility-bar|" +
+	"^vestpocket$|" +
+	"vcard|" +
+	"^watch-action-panels$|" +
+	"^watch-discussion$|" +
+	"welcome_form|" +
+	"^whats[_-]next$|" +
+	"wp-caption-text")
+
+// Clean removes HTML elements around the main content and prepares the document for parsing
+func (c *Cleaner) Clean(docToClean *goquery.Document) *goquery.Document {
+	if c.config.debug {
+		log.Println("Starting cleaning phase with Cleaner")
+	}
+	docToClean = c.cleanBr(docToClean)
+	docToClean = c.cleanArticleTags(docToClean)
+	docToClean = c.cleanEMTags(docToClean)
+	docToClean = c.dropCaps(docToClean)
+	docToClean = c.removeScriptsStyle(docToClean)
+	docToClean = c.cleanBadTags(docToClean, removeNodesRegEx, &[]string{"id", "class", "name"})
+	docToClean = c.cleanBadTags(docToClean, removeVisibilityStyleRegEx, &[]string{"style"})
+	docToClean = c.removeTags(docToClean, &[]string{"nav", "footer", "aside", "cite"})
+	docToClean = c.cleanParaSpans(docToClean)
+
+
+	docToClean = c.convertDivsToParagraphs(docToClean, "div")
+
+	docToClean = c.convertDivsToParagraphs(docToClean, "span")
+	docToClean = c.convertDivsToParagraphs(docToClean, "article")
+	docToClean = c.convertDivsToParagraphs(docToClean, "pre")
+
+	return docToClean
+}
+
+func (c *Cleaner) cleanArticleTags(doc *goquery.Document) *goquery.Document {
+	tags := [3]string{"id", "name", "class"}
+	articles := doc.Find("article")
+	articles.Each(func(i int, s *goquery.Selection) {
+		for _, tag := range tags {
+			c.config.parser.delAttr(s, tag)
+		}
+	})
+	return doc
+}
+
+// replace <br /> with \n\n
+func (c *Cleaner) cleanBr(doc *goquery.Document) *goquery.Document {
+	linebreaks := doc.Find("br")
+	linebreaks.Each(func(i int, br *goquery.Selection) {
+		node := br.Get(0)
+		node.Data = "\n\n"
+		node.Type = html.TextNode
+		node.Attr = []html.Attribute{}
+		node.DataAtom = 0
+		node.FirstChild = nil
+		node.LastChild = nil
+	})
+	return doc
+}
+
+func (c *Cleaner) cleanEMTags(doc *goquery.Document) *goquery.Document {
+	ems := doc.Find("em")
+	ems.Each(func(i int, s *goquery.Selection) {
+		images := s.Find("img")
+		if images.Length() == 0 {
+			c.config.parser.dropTag(s)
+		}
+	})
+	if c.config.debug {
+		log.Printf("Cleaning %d EM tags\n", ems.Size())
+	}
+	return doc
+}
+
+func (c *Cleaner) removeTags(doc *goquery.Document, tags *[]string) *goquery.Document {
+	for _, tag := range *tags {
+		node := doc.Find(tag)
+		node.Each(func(i int, s *goquery.Selection) {
+			c.config.parser.removeNode(s)
+		})
+	}
+	return doc
+}
+
+func (c *Cleaner) cleanDivs(doc *goquery.Document) *goquery.Document {
+	frames := make(map[string]int)
+	framesNodes := make(map[string]*list.List)
+	divs := doc.Find("div")
+	divs.Each(func(i int, s *goquery.Selection) {
+		children := s.Children()
+		if children.Size() == 0 {
+			text := strings.Trim(s.Text(), " \t")
+			text = strings.ToLower(text)
+			frames[text]++
+			if framesNodes[text] == nil {
+				framesNodes[text] = list.New()
+			}
+			framesNodes[text].PushBack(s)
+		}
+	})
+	for text, freq := range frames {
+		if freq > 1 {
+			selections := framesNodes[text]
+			for s := selections.Front(); s != nil; s = s.Next() {
+				selection := s.Value.(*goquery.Selection)
+				c.config.parser.removeNode(selection)
+			}
+		}
+	}
+	return doc
+}
+
+func (c *Cleaner) dropCaps(doc *goquery.Document) *goquery.Document {
+	items := doc.Find("span")
+	count := 0 // remove
+	items.Each(func(i int, s *goquery.Selection) {
+		attribute, exists := s.Attr("class")
+		if exists && (strings.Contains(attribute, "dropcap") || strings.Contains(attribute, "drop_cap")) {
+			c.config.parser.dropTag(s)
+			count++
+		}
+	})
+	if c.config.debug && count > 0 {
+		log.Printf("Cleaned %d dropcap tags\n", count)
+	}
+	return doc
+}
+
+func (c *Cleaner) removeScriptsStyle(doc *goquery.Document) *goquery.Document {
+	if c.config.debug {
+		log.Println("Starting to remove script tags")
+	}
+	count := 0 // number of removed nodes
+	scripts := doc.Find("script,noscript,style")
+	scripts.Each(func(i int, s *goquery.Selection) {
+		c.config.parser.removeNode(s)
+		count++
+	})
+	if c.config.debug && count > 0 {
+		log.Printf("Removed %d script and style tags\n", scripts.Size())
+	}
+	return doc
+}
+
+func (c *Cleaner) cleanBadTags(doc *goquery.Document, pattern *regexp.Regexp, selectors *[]string) *goquery.Document {
+	body := doc.Find("html")
+	children := body.Children()
+	children.Each(func(i int, s *goquery.Selection) {
+		for _, selector := range *selectors {
+			naughtyList := s.Find("*[" + selector + "]")
+			count := 0
+			naughtyList.Each(func(j int, node *goquery.Selection) {
+				attribute, _ := node.Attr(selector)
+				if pattern.MatchString(attribute) {
+					if c.config.debug {
+						log.Printf("Cleaning: Removing node with %s: %s\n", selector, c.config.parser.name(selector, node))
+					}
+					c.config.parser.removeNode(node)
+					count++
+				}
+			})
+			if c.config.debug && count > 0 {
+				log.Printf("%d naughty %s elements found", count, selector)
+			}
+		}
+	})
+	return doc
+}
+
+// Replace <p><span>...</span></p>   with  <p>...</p>
+func (c *Cleaner) cleanParaSpans(doc *goquery.Document) *goquery.Document {
+	spans := doc.Find("span")
+	spans.Each(func(i int, s *goquery.Selection) {
+		parent := s.Parent()
+		if parent != nil && parent.Length() > 0 && parent.Get(0).DataAtom == atom.P {
+			replaceTagWithContents(s, whitelistedTextAtomTypes)
+		}
+	})
+	return doc
+}
+
+func (c *Cleaner) getFlushedBuffer(fragment string) []*html.Node {
+	var output []*html.Node
+	reader := strings.NewReader(fragment)
+	document, _ := html.Parse(reader)
+	body := document.FirstChild.LastChild
+	for c := body.FirstChild; c != nil; c = c.NextSibling {
+		output = append(output, c)
+		c.Parent = nil
+		c.PrevSibling = nil
+	}
+
+	for _, o := range output {
+		o.NextSibling = nil
+	}
+	return output
+}
+
+func (c *Cleaner) replaceWithPara(div *goquery.Selection) {
+	if div.Size() > 0 {
+		node := div.Get(0)
+		node.Data = atom.P.String()
+		node.DataAtom = atom.P
+		node.Attr = []html.Attribute{}
+	}
+}
+
+func (c *Cleaner) tabsAndNewLinesReplacements(text string) string {
+	text = strings.Replace(text, "\n", "\n\n", -1)
+	text = tabsRegEx.ReplaceAllString(text, "")
+	return text
+}
+
+func (c *Cleaner) convertDivsToParagraphs(doc *goquery.Document, domType string) *goquery.Document {
+	if c.config.debug {
+		log.Println("Starting to replace bad divs...")
+	}
+	badDivs := 0
+	convertedTextNodes := 0
+	divs := doc.Find(domType)
+
+	divs.Each(func(i int, div *goquery.Selection) {
+		divHTML, _ := div.Html()
+		if divToPElementsPattern.Match([]byte(divHTML)) {
+			c.replaceWithPara(div)
+			badDivs++
+		} else {
+			var replacementText []string
+			nodesToRemove := list.New()
+			children := div.Contents()
+			if c.config.debug {
+				log.Printf("Found %d children of div\n", children.Size())
+			}
+			children.EachWithBreak(func(i int, kid *goquery.Selection) bool {
+				text := kid.Text()
+				kidNode := kid.Get(0)
+				tag := kidNode.Data
+				if tag == text {
+					tag = "#text"
+				}
+				if tag == "#text" {
+					text = strings.Replace(text, "\n", "", -1)
+					text = tabsRegEx.ReplaceAllString(text, "")
+					if text == "" {
+						return true
+					}
+					if len(text) > 1 {
+						prev := kidNode.PrevSibling
+						if c.config.debug {
+							log.Printf("PARENT CLASS: %s NODENAME: %s\n", c.config.parser.name("class", div), tag)
+							log.Printf("TEXTREPLACE: %s\n", strings.Replace(text, "\n", "", -1))
+						}
+						if prev != nil && prev.DataAtom == atom.A {
+							nodeSelection := kid.HasNodes(prev)
+							html, _ := nodeSelection.Html()
+							replacementText = append(replacementText, html)
+							if c.config.debug {
+								log.Printf("SIBLING NODENAME ADDITION: %s TEXT: %s\n", prev.Data, html)
+							}
+						}
+						replacementText = append(replacementText, text)
+						nodesToRemove.PushBack(kidNode)
+						convertedTextNodes++
+					}
+
+				}
+				return true
+			})
+
+			/*
+			newNode := new(html.Node)
+			newNode.Type = html.ElementNode
+			newNode.Data = strings.Join(replacementText, "")
+			newNode.DataAtom = atom.P
+			*/
+/*
+			replacementText = strings.Replace(replacementText, "=C3=A8", "è")
+			replacementText = strings.Replace(replacementText, "=C3=A9", "é")
+*/
+			div.First().BeforeHtml("<p>" + strings.Join(replacementText, "") + "</p>")
+
+			for s := nodesToRemove.Front(); s != nil; s = s.Next() {
+				node := s.Value.(*html.Node)
+				if node != nil && node.Parent != nil {
+					node.Parent.RemoveChild(node)
+				}
+			}
+		}
+	})
+	if c.config.debug {
+		log.Printf("Found %d total divs with %d bad divs replaced and %d textnodes converted inside divs", divs.Size(), badDivs, convertedTextNodes)
+	}
+	return doc
+
+}
--- a/vendor/github.com/advancedlogic/GoOse/configuration.go
+++ b/vendor/github.com/advancedlogic/GoOse/configuration.go
@@ -0,0 +1,69 @@
+package goose
+
+import (
+	"time"
+)
+
+const defaultUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/534.52.7 (KHTML, like Gecko) Version/5.1.2 Safari/534.52.7"
+
+// Configuration is a wrapper for various config options
+type Configuration struct {
+	localStoragePath        string //not used in this version
+	imagesMinBytes          int    //not used in this version
+	targetLanguage          string
+	imageMagickConvertPath  string //not used in this version
+	imageMagickIdentifyPath string //not used in this version
+	browserUserAgent        string
+	debug                   bool
+	extractPublishDate      bool
+	additionalDataExtractor bool
+	enableImageFetching     bool
+	useMetaLanguage         bool
+
+	//path to the stopwords folder
+	stopWordsPath string
+	stopWords     StopWords
+	parser        *Parser
+
+	timeout time.Duration
+}
+
+// GetDefaultConfiguration returns safe default configuration options
+func GetDefaultConfiguration(args ...string) Configuration {
+	if len(args) == 0 {
+		return Configuration{
+			localStoragePath:        "",   //not used in this version
+			imagesMinBytes:          4500, //not used in this version
+			enableImageFetching:     true,
+			useMetaLanguage:         true,
+			targetLanguage:          "en",
+			imageMagickConvertPath:  "/usr/bin/convert",  //not used in this version
+			imageMagickIdentifyPath: "/usr/bin/identify", //not used in this version
+			browserUserAgent:        defaultUserAgent,
+			debug:                   false,
+			extractPublishDate:      true,
+			additionalDataExtractor: false,
+			stopWordsPath:           "resources/stopwords",
+			stopWords:               NewStopwords(), //TODO with path
+			parser:                  NewParser(),
+			timeout:                 time.Duration(5 * time.Second),
+		}
+	}
+	return Configuration{
+		localStoragePath:        "",   //not used in this version
+		imagesMinBytes:          4500, //not used in this version
+		enableImageFetching:     true,
+		useMetaLanguage:         true,
+		targetLanguage:          "en",
+		imageMagickConvertPath:  "/usr/bin/convert",  //not used in this version
+		imageMagickIdentifyPath: "/usr/bin/identify", //not used in this version
+		browserUserAgent:        defaultUserAgent,
+		debug:                   false,
+		extractPublishDate:      true,
+		additionalDataExtractor: false,
+		stopWordsPath:           "resources/stopwords",
+		stopWords:               NewStopwords(), //TODO with path
+		parser:                  NewParser(),
+		timeout:                 time.Duration(5 * time.Second),
+	}
+}
--- a/vendor/github.com/advancedlogic/GoOse/coverage.sh
+++ b/vendor/github.com/advancedlogic/GoOse/coverage.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# Run test coverage on each subdirectory and merge the coverage profile.
+echo "mode: count" > target/report/profile.cov
+
+# Standard go tooling behavior is to ignore dirs with leading underscors
+for dir in $(find . -maxdepth 10 -not -path './.git*' -not -path '*/_*' -type d); do
+    if ls $dir/*.go &> /dev/null; then
+        go test -covermode=count -coverprofile=$dir/profile.tmp $dir
+        if [ -f $dir/profile.tmp ]; then
+            cat $dir/profile.tmp | tail -n +2 >> target/report/profile.cov
+            rm $dir/profile.tmp
+        fi
+    fi
+done
+go tool cover -html target/report/profile.cov -o target/report/coverage.html
+
--- a/vendor/github.com/advancedlogic/GoOse/crawler.go
+++ b/vendor/github.com/advancedlogic/GoOse/crawler.go
@@ -0,0 +1,180 @@
+package goose
+
+import (
+	"errors"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+// Crawler can fetch the target HTML page
+type Crawler struct {
+	config  Configuration
+	Charset string
+}
+
+// NewCrawler returns a crawler object initialised with the URL and the [optional] raw HTML body
+func NewCrawler(config Configuration) Crawler {
+	return Crawler{
+		config:  config,
+		Charset: "",
+	}
+}
+
+func getCharsetFromContentType(cs string) string {
+	cs = strings.ToLower(strings.Replace(cs, " ", "", -1))
+	cs = strings.TrimPrefix(cs, "text/html;charset=")
+	cs = strings.TrimPrefix(cs, "text/xhtml;charset=")
+	cs = strings.TrimPrefix(cs, "application/xhtml+xml;charset=")
+	return NormaliseCharset(cs)
+}
+
+// SetCharset can be used to force a charset (e.g. when read from the HTTP headers)
+// rather than relying on the detection from the HTML meta tags
+func (c *Crawler) SetCharset(cs string) {
+	c.Charset = getCharsetFromContentType(cs)
+}
+
+// GetContentType returns the Content-Type string extracted from the meta tags
+func (c Crawler) GetContentType(document *goquery.Document) string {
+	var attr string
+	// <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+	document.Find("meta[http-equiv#=(?i)^Content\\-type$]").Each(func(i int, s *goquery.Selection) {
+		attr, _ = s.Attr("content")
+	})
+	return attr
+}
+
+// GetCharset returns a normalised charset string extracted from the meta tags
+func (c Crawler) GetCharset(document *goquery.Document) string {
+	// manually-provided charset (from HTTP headers?) takes priority
+	if "" != c.Charset {
+		return c.Charset
+	}
+
+	// <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+	ct := c.GetContentType(document)
+	if "" != ct && strings.Contains(strings.ToLower(ct), "charset") {
+		return getCharsetFromContentType(ct)
+	}
+
+	// <meta charset="utf-8">
+	selection := document.Find("meta").EachWithBreak(func(i int, s *goquery.Selection) bool {
+		_, exists := s.Attr("charset")
+		return !exists
+	})
+
+	if selection != nil {
+		cs, _ := selection.Attr("charset")
+		return NormaliseCharset(cs)
+	}
+
+	return ""
+}
+
+// Preprocess fetches the HTML page if needed, converts it to UTF-8 and applies
+// some text normalisation to guarantee better results when extracting the content
+func (c *Crawler) Preprocess(RawHTML string) (*goquery.Document, error) {
+	var err error
+
+	if RawHTML == "" {
+		return nil, errors.New("cannot process empty HTML content")
+	}
+
+	RawHTML = c.addSpacesBetweenTags(RawHTML)
+
+	reader := strings.NewReader(RawHTML)
+	document, err := goquery.NewDocumentFromReader(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	cs := c.GetCharset(document)
+	//log.Println("-------------------------------------------CHARSET:", cs)
+	if "" != cs && "UTF-8" != cs {
+		// the net/html parser and goquery require UTF-8 data
+		RawHTML = UTF8encode(RawHTML, cs)
+		reader = strings.NewReader(RawHTML)
+		if document, err = goquery.NewDocumentFromReader(reader); err != nil {
+			return nil, err
+		}
+	}
+
+	return document, nil
+}
+
+// Crawl fetches the HTML body and returns an Article
+func (c Crawler) Crawl(RawHTML string, url string) (*Article, error) {
+	article := new(Article)
+
+	document, err := c.Preprocess(RawHTML)
+	if nil != err {
+		return nil, err
+	}
+	if nil == document {
+		return article, nil
+	}
+	extractor := NewExtractor(c.config)
+	startTime := time.Now().UnixNano()
+
+	article.RawHTML, err = document.Html()
+	if nil != err {
+		return nil, err
+	}
+	article.FinalURL = url
+	article.Doc = document
+
+	article.TitleUnmodified = extractor.getTitleUnmodified(document)
+	article.Title = extractor.GetTitleFromUnmodifiedTitle(article.TitleUnmodified)
+	article.MetaLang = extractor.GetMetaLanguage(document)
+	article.MetaFavicon = extractor.GetFavicon(document)
+
+	article.MetaDescription = extractor.GetMetaContentWithSelector(document, "meta[name#=(?i)^description$]")
+	article.MetaKeywords = extractor.GetMetaContentWithSelector(document, "meta[name#=(?i)^keywords$]")
+	article.CanonicalLink = extractor.GetCanonicalLink(document)
+	if "" == article.CanonicalLink {
+		article.CanonicalLink = article.FinalURL
+	}
+	article.Domain = extractor.GetDomain(article.CanonicalLink)
+	article.Tags = extractor.GetTags(document)
+
+	if c.config.extractPublishDate {
+		if timestamp := extractor.GetPublishDate(document); timestamp != nil {
+			article.PublishDate = timestamp
+		}
+	}
+
+	cleaner := NewCleaner(c.config)
+	article.Doc = cleaner.Clean(article.Doc)
+
+	article.TopImage = OpenGraphResolver(document)
+	if article.TopImage == "" {
+		article.TopImage = WebPageResolver(article)
+	}
+
+	article.TopNode = extractor.CalculateBestNode(document)
+	if article.TopNode != nil {
+		article.TopNode = extractor.PostCleanup(article.TopNode)
+
+		article.CleanedText, article.Links = extractor.GetCleanTextAndLinks(article.TopNode, article.MetaLang)
+
+		videoExtractor := NewVideoExtractor()
+		article.Movies = videoExtractor.GetVideos(document)
+	}
+
+	article.Delta = time.Now().UnixNano() - startTime
+
+	return article, nil
+}
+
+// In many cases, like at the end of each <li> element or between </span><span> tags,
+// we need to add spaces, otherwise the text on either side will get joined together into one word.
+// This method also adds newlines after each </p> tag to preserve paragraphs.
+func (c Crawler) addSpacesBetweenTags(text string) string {
+	text = strings.Replace(text, "><", "> <", -1)
+	text = strings.Replace(text, "</blockquote>", "</blockquote>\n", -1)
+	text = strings.Replace(text, "<img ", "\n<img ", -1)
+	text = strings.Replace(text, "</li>", "</li>\n", -1)
+	return strings.Replace(text, "</p>", "</p>\n", -1)
+}
--- a/vendor/github.com/advancedlogic/GoOse/crawlershort.go
+++ b/vendor/github.com/advancedlogic/GoOse/crawlershort.go
@@ -0,0 +1,155 @@
+package goose
+
+import (
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/pkg/errors"
+)
+
+// Crawler can fetch the target HTML page
+type CrawlerShort struct {
+	config  Configuration
+	Charset string
+}
+
+// NewCrawler returns a crawler object initialised with the URL and the [optional] raw HTML body
+func NewCrawlerShort(config Configuration) CrawlerShort {
+	return CrawlerShort{
+		config:  config,
+		Charset: "",
+	}
+}
+
+// SetCharset can be used to force a charset (e.g. when read from the HTTP headers)
+// rather than relying on the detection from the HTML meta tags
+func (c *CrawlerShort) SetCharset(cs string) {
+	c.Charset = getCharsetFromContentType(cs)
+}
+
+// GetContentType returns the Content-Type string extracted from the meta tags
+func (c CrawlerShort) GetContentType(document *goquery.Document) string {
+	var attr string
+	// <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+	document.Find("meta[http-equiv#=(?i)^Content\\-type$]").Each(func(i int, s *goquery.Selection) {
+		attr, _ = s.Attr("content")
+	})
+	return attr
+}
+
+// GetCharset returns a normalised charset string extracted from the meta tags
+func (c CrawlerShort) GetCharset(document *goquery.Document) string {
+	// manually-provided charset (from HTTP headers?) takes priority
+	if "" != c.Charset {
+		return c.Charset
+	}
+
+	// <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+	ct := c.GetContentType(document)
+	if "" != ct && strings.Contains(strings.ToLower(ct), "charset") {
+		return getCharsetFromContentType(ct)
+	}
+
+	// <meta charset="utf-8">
+	selection := document.Find("meta").EachWithBreak(func(i int, s *goquery.Selection) bool {
+		_, exists := s.Attr("charset")
+		return !exists
+	})
+
+	if selection != nil {
+		cs, _ := selection.Attr("charset")
+		return NormaliseCharset(cs)
+	}
+
+	return ""
+}
+
+// Preprocess fetches the HTML page if needed, converts it to UTF-8 and applies
+// some text normalisation to guarantee better results when extracting the content
+func (c *CrawlerShort) Preprocess(RawHTML string) (*goquery.Document, error) {
+	var err error
+
+	RawHTML = c.addSpacesBetweenTags(RawHTML)
+
+	reader := strings.NewReader(RawHTML)
+	document, err := goquery.NewDocumentFromReader(reader)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not perform goquery.NewDocumentFromReader(reader)")
+	}
+
+	cs := c.GetCharset(document)
+	//log.Println("-------------------------------------------CHARSET:", cs)
+	if "" != cs && "UTF-8" != cs {
+		// the net/html parser and goquery require UTF-8 data
+		RawHTML = UTF8encode(RawHTML, cs)
+		reader = strings.NewReader(RawHTML)
+		if document, err = goquery.NewDocumentFromReader(reader); err != nil {
+			return nil, errors.Wrap(err, "could not perform goquery.NewDocumentFromReader(reader)")
+		}
+	}
+
+	return document, nil
+}
+
+// Crawl fetches the HTML body and returns an Article
+func (c CrawlerShort) Crawl(RawHTML, url string) (*Article, error) {
+	article := new(Article)
+
+	document, err := c.Preprocess(RawHTML)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not Preprocess RawHTML")
+	}
+	if document == nil {
+		return article, nil
+	}
+
+	extractor := NewExtractor(c.config)
+
+	startTime := time.Now().UnixNano()
+
+	article.RawHTML, err = document.Html()
+	if err != nil {
+		return nil, errors.Wrap(err, "could not get html from document")
+	}
+	article.FinalURL = url
+
+	article.Title = extractor.GetTitle(document)
+	article.MetaDescription = extractor.GetMetaContentWithSelector(document, "meta[name#=(?i)^description$]")
+
+	if c.config.extractPublishDate {
+		if timestamp := extractor.GetPublishDate(document); timestamp != nil {
+			article.PublishDate = timestamp
+		}
+	}
+
+	cleaner := NewCleaner(c.config)
+	article.Doc = cleaner.Clean(article.Doc)
+
+	article.TopImage = OpenGraphResolver(document)
+	if article.TopImage == "" {
+		article.TopImage = WebPageResolver(article)
+	}
+
+	article.TopNode = extractor.CalculateBestNode(document)
+	if article.TopNode != nil {
+		article.TopNode = extractor.PostCleanup(article.TopNode)
+
+		article.CleanedText, article.Links = extractor.GetCleanTextAndLinks(article.TopNode, article.MetaLang)
+
+	}
+	article.Delta = time.Now().UnixNano() - startTime
+
+	return article, nil
+}
+
+// In many cases, like at the end of each <li> element or between </span><span> tags,
+// we need to add spaces, otherwise the text on either side will get joined together into one word.
+// This method also adds newlines after each </p> tag to preserve paragraphs.
+func (c CrawlerShort) addSpacesBetweenTags(text string) string {
+	text = strings.Replace(text, "><", "> <", -1)
+	text = strings.Replace(text, "</blockquote>", "</blockquote>\n", -1)
+	text = strings.Replace(text, "<img ", "\n<img ", -1)
+	text = strings.Replace(text, "</li>", "</li>\n", -1)
+	return strings.Replace(text, "</p>", "</p>\n", -1)
+}
--- a/vendor/github.com/advancedlogic/GoOse/doc.go
+++ b/vendor/github.com/advancedlogic/GoOse/doc.go
@@ -0,0 +1,23 @@
+/*
+This is a golang port of "Goose" originally licensed to Gravity.com
+under one or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.
+
+Golang port was written by Antonio Linari
+
+Gravity.com licenses this file
+to you under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package goose
--- a/vendor/github.com/advancedlogic/GoOse/extractor.go
+++ b/vendor/github.com/advancedlogic/GoOse/extractor.go
@@ -0,0 +1,754 @@
+package goose
+
+import (
+	"container/list"
+	"log"
+	"math"
+	"net/url"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/araddon/dateparse"
+	"github.com/fatih/set"
+	"github.com/gigawattio/window"
+	"github.com/jaytaylor/html2text"
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/atom"
+)
+
+const defaultLanguage = "en"
+
+var motleyReplacement = "&#65533;" // U+FFFD (decimal 65533) is the "replacement character".
+//var escapedFragmentReplacement = regexp.MustCompile("#!")
+//var titleReplacements = regexp.MustCompile("&raquo;")
+
+var titleDelimiters = []string{
+	"|",
+	" - ",
+	" — ",
+	"»",
+	":",
+}
+
+var aRelTagSelector = "a[rel=tag]"
+var aHrefTagSelector = [...]string{"/tag/", "/tags/", "/topic/", "?keyword"}
+
+//var langRegEx = "^[A-Za-z]{2}$"
+
+// ContentExtractor can parse the HTML and fetch various properties
+type ContentExtractor struct {
+	config Configuration
+}
+
+// NewExtractor returns a configured HTML parser
+func NewExtractor(config Configuration) ContentExtractor {
+	return ContentExtractor{
+		config: config,
+	}
+}
+
+//if the article has a title set in the source, use that
+func (extr *ContentExtractor) getTitleUnmodified(document *goquery.Document) string {
+	title := ""
+
+	titleElement := document.Find("title")
+	if titleElement != nil && titleElement.Size() > 0 {
+		title = titleElement.Text()
+	}
+
+	if title == "" {
+		ogTitleElement := document.Find(`meta[property="og:title"]`)
+		if ogTitleElement != nil && ogTitleElement.Size() > 0 {
+			title, _ = ogTitleElement.Attr("content")
+		}
+	}
+
+	if title == "" {
+		titleElement = document.Find("post-title,headline")
+		if titleElement == nil || titleElement.Size() == 0 {
+			return title
+		}
+		title = titleElement.Text()
+	}
+	return title
+}
+
+// GetTitleFromUnmodifiedTitle returns the title from the unmodified one
+func (extr *ContentExtractor) GetTitleFromUnmodifiedTitle(title string) string {
+	for _, delimiter := range titleDelimiters {
+		if strings.Contains(title, delimiter) {
+			title = extr.splitTitle(strings.Split(title, delimiter))
+			break
+		}
+	}
+
+	title = strings.Replace(title, motleyReplacement, "", -1)
+
+	if extr.config.debug {
+		log.Printf("Page title is %s\n", title)
+	}
+
+	return strings.TrimSpace(title)
+}
+
+// GetTitle returns the title set in the source, if the article has one
+func (extr *ContentExtractor) GetTitle(document *goquery.Document) string {
+	title := extr.getTitleUnmodified(document)
+	return extr.GetTitleFromUnmodifiedTitle(title)
+}
+
+func (extr *ContentExtractor) splitTitle(titles []string) string {
+	largeTextLength := 0
+	largeTextIndex := 0
+	for i, current := range titles {
+		if len(current) > largeTextLength {
+			largeTextLength = len(current)
+			largeTextIndex = i
+		}
+	}
+	title := titles[largeTextIndex]
+	title = strings.Replace(title, "&raquo;", "»", -1)
+	return title
+}
+
+// GetMetaLanguage returns the meta language set in the source, if the article has one
+func (extr *ContentExtractor) GetMetaLanguage(document *goquery.Document) string {
+	var language string
+	shtml := document.Find("html")
+	attr, _ := shtml.Attr("lang")
+	if attr == "" {
+		attr, _ = document.Attr("lang")
+	}
+	if attr == "" {
+		selection := document.Find("meta").EachWithBreak(func(i int, s *goquery.Selection) bool {
+			var exists bool
+			attr, exists = s.Attr("http-equiv")
+			if exists && attr == "content-language" {
+				return false
+			}
+			return true
+		})
+		if selection != nil {
+			attr, _ = selection.Attr("content")
+		}
+	}
+	idx := strings.LastIndex(attr, "-")
+	if idx == -1 {
+		language = attr
+	} else {
+		language = attr[0:idx]
+	}
+
+	_, ok := sw[language]
+
+	if language == "" || !ok {
+		language = extr.config.stopWords.SimpleLanguageDetector(shtml.Text())
+		if language == "" {
+			language = defaultLanguage
+		}
+	}
+
+	extr.config.targetLanguage = language
+	return language
+}
+
+// GetFavicon returns the favicon set in the source, if the article has one
+func (extr *ContentExtractor) GetFavicon(document *goquery.Document) string {
+	favicon := ""
+	document.Find("link").EachWithBreak(func(i int, s *goquery.Selection) bool {
+		attr, exists := s.Attr("rel")
+		if exists && strings.Contains(attr, "icon") {
+			favicon, _ = s.Attr("href")
+			return false
+		}
+		return true
+	})
+	return favicon
+}
+
+// GetMetaContentWithSelector returns the content attribute of meta tag matching the selector
+func (extr *ContentExtractor) GetMetaContentWithSelector(document *goquery.Document, selector string) string {
+	selection := document.Find(selector)
+	content, _ := selection.Attr("content")
+	return strings.TrimSpace(content)
+}
+
+// GetMetaContent returns the content attribute of meta tag with the given property name
+func (extr *ContentExtractor) GetMetaContent(document *goquery.Document, metaName string) string {
+	content := ""
+	document.Find("meta").EachWithBreak(func(i int, s *goquery.Selection) bool {
+		attr, exists := s.Attr("name")
+		if exists && attr == metaName {
+			content, _ = s.Attr("content")
+			return false
+		}
+		attr, exists = s.Attr("itemprop")
+		if exists && attr == metaName {
+			content, _ = s.Attr("content")
+			return false
+		}
+		return true
+	})
+	return content
+}
+
+// GetMetaContents returns all the meta tags as name->content pairs
+func (extr *ContentExtractor) GetMetaContents(document *goquery.Document, metaNames *set.Set) map[string]string {
+	contents := make(map[string]string)
+	counter := metaNames.Size()
+	document.Find("meta").EachWithBreak(func(i int, s *goquery.Selection) bool {
+		attr, exists := s.Attr("name")
+		if exists && metaNames.Has(attr) {
+			content, _ := s.Attr("content")
+			contents[attr] = content
+			counter--
+			if counter < 0 {
+				return false
+			}
+		}
+		return true
+	})
+	return contents
+}
+
+// GetMetaDescription returns the meta description set in the source, if the article has one
+func (extr *ContentExtractor) GetMetaDescription(document *goquery.Document) string {
+	return extr.GetMetaContent(document, "description")
+}
+
+// GetMetaKeywords returns the meta keywords set in the source, if the article has them
+func (extr *ContentExtractor) GetMetaKeywords(document *goquery.Document) string {
+	return extr.GetMetaContent(document, "keywords")
+}
+
+// GetMetaAuthor returns the meta author set in the source, if the article has one
+func (extr *ContentExtractor) GetMetaAuthor(document *goquery.Document) string {
+	return extr.GetMetaContent(document, "author")
+}
+
+// GetMetaContentLocation returns the meta content location set in the source, if the article has one
+func (extr *ContentExtractor) GetMetaContentLocation(document *goquery.Document) string {
+	return extr.GetMetaContent(document, "contentLocation")
+}
+
+// GetCanonicalLink returns the meta canonical link set in the source
+func (extr *ContentExtractor) GetCanonicalLink(document *goquery.Document) string {
+	metas := document.Find("link[rel=canonical]")
+	if metas.Length() > 0 {
+		meta := metas.First()
+		href, _ := meta.Attr("href")
+		href = strings.Trim(href, "\n")
+		href = strings.Trim(href, " ")
+		if href != "" {
+			return href
+		}
+	}
+	return ""
+}
+
+// GetDomain extracts the domain from a link
+func (extr *ContentExtractor) GetDomain(canonicalLink string) string {
+	u, err := url.Parse(canonicalLink)
+	if err == nil {
+		return u.Host
+	}
+	return ""
+}
+
+// GetTags returns the tags set in the source, if the article has them
+func (extr *ContentExtractor) GetTags(document *goquery.Document) *set.Set {
+	tags := set.New(set.ThreadSafe).(*set.Set)
+	selections := document.Find(aRelTagSelector)
+	selections.Each(func(i int, s *goquery.Selection) {
+		tags.Add(s.Text())
+	})
+	selections = document.Find("a")
+	selections.Each(func(i int, s *goquery.Selection) {
+		href, exists := s.Attr("href")
+		if exists {
+			for _, part := range aHrefTagSelector {
+				if strings.Contains(href, part) {
+					tags.Add(s.Text())
+				}
+			}
+		}
+	})
+
+	return tags
+}
+
+// GetPublishDate returns the publication date, if one can be located.
+func (extr *ContentExtractor) GetPublishDate(document *goquery.Document) *time.Time {
+	raw, err := document.Html()
+	if err != nil {
+		log.Printf("Error converting document HTML nodes to raw HTML: %s (publish date detection aborted)\n", err)
+		return nil
+	}
+
+	text, err := html2text.FromString(raw)
+	if err != nil {
+		log.Printf("Error converting document HTML to plaintext: %s (publish date detection aborted)\n", err)
+		return nil
+	}
+
+	text = strings.ToLower(text)
+
+	// Simplify months because the dateparse pkg only handles abbreviated.
+	for k, v := range map[string]string{
+		"january":  "jan",
+		"march":    "mar",
+		"february": "feb",
+		"april":    "apr",
+		// "may":       "may", // Pointless.
+		"june":      "jun",
+		"august":    "aug",
+		"september": "sep",
+		"sept":      "sep",
+		"october":   "oct",
+		"november":  "nov",
+		"december":  "dec",
+		"th,":       ",", // Strip day number suffixes.
+		"rd,":       ",",
+	} {
+		text = strings.Replace(text, k, v, -1)
+	}
+	text = strings.Replace(text, "\n", " ", -1)
+	text = regexp.MustCompile(" +").ReplaceAllString(text, " ")
+
+	tuple1 := strings.Split(text, " ")
+
+	var (
+		expr  = regexp.MustCompile("[0-9]")
+		ts    time.Time
+		found bool
+	)
+	for _, n := range []int{3, 4, 5, 2, 6} {
+		for _, win := range window.Rolling(tuple1, n) {
+			if !expr.MatchString(strings.Join(win, " ")) {
+				continue
+			}
+
+			input := strings.Join(win, " ")
+			ts, err = dateparse.ParseAny(input)
+			if err == nil && ts.Year() > 0 && ts.Month() > 0 && ts.Day() > 0 {
+				found = true
+				break
+			}
+
+			// Try injecting a comma for dateparse.
+			win[1] = win[1] + ","
+			input = strings.Join(win, " ")
+			ts, err = dateparse.ParseAny(input)
+			if err == nil && ts.Year() > 0 && ts.Month() > 0 && ts.Day() > 0 {
+				found = true
+				break
+			}
+		}
+		if found {
+			break
+		}
+	}
+
+	if found {
+		return &ts
+	}
+	return nil
+}
+
+// GetCleanTextAndLinks parses the main HTML node for text and links
+func (extr *ContentExtractor) GetCleanTextAndLinks(topNode *goquery.Selection, lang string) (string, []string) {
+	outputFormatter := new(outputFormatter)
+	outputFormatter.config = extr.config
+	return outputFormatter.getFormattedText(topNode, lang)
+}
+
+// CalculateBestNode checks for the HTML node most likely to contain the main content.
+//we're going to start looking for where the clusters of paragraphs are. We'll score a cluster based on the number of stopwords
+//and the number of consecutive paragraphs together, which should form the cluster of text that this node is around
+//also store on how high up the paragraphs are, comments are usually at the bottom and should get a lower score
+func (extr *ContentExtractor) CalculateBestNode(document *goquery.Document) *goquery.Selection {
+	var topNode *goquery.Selection
+	nodesToCheck := extr.nodesToCheck(document)
+	if extr.config.debug {
+		log.Printf("Nodes to check %d\n", len(nodesToCheck))
+	}
+	startingBoost := 1.0
+	cnt := 0
+	i := 0
+	parentNodes := set.New(set.ThreadSafe).(*set.Set)
+	nodesWithText := list.New()
+	for _, node := range nodesToCheck {
+		textNode := node.Text()
+		ws := extr.config.stopWords.stopWordsCount(extr.config.targetLanguage, textNode)
+		highLinkDensity := extr.isHighLinkDensity(node)
+		if ws.stopWordCount > 2 && !highLinkDensity {
+			nodesWithText.PushBack(node)
+		}
+	}
+	nodesNumber := nodesWithText.Len()
+	negativeScoring := 0
+	bottomNegativeScoring := float64(nodesNumber) * 0.25
+
+	if extr.config.debug {
+		log.Printf("About to inspect num of nodes with text %d\n", nodesNumber)
+	}
+
+	for n := nodesWithText.Front(); n != nil; n = n.Next() {
+		node := n.Value.(*goquery.Selection)
+		boostScore := 0.0
+		if extr.isBoostable(node) {
+			if cnt >= 0 {
+				boostScore = float64((1.0 / startingBoost) * 50)
+				startingBoost++
+			}
+		}
+
+		if nodesNumber > 15 {
+			if float64(nodesNumber-i) <= bottomNegativeScoring {
+				booster := bottomNegativeScoring - float64(nodesNumber-i)
+				boostScore = -math.Pow(booster, 2.0)
+				negScore := math.Abs(boostScore) + float64(negativeScoring)
+				if negScore > 40 {
+					boostScore = 5.0
+				}
+			}
+		}
+
+		if extr.config.debug {
+			log.Printf("Location Boost Score %1.5f on iteration %d id='%s' class='%s'\n", boostScore, i, extr.config.parser.name("id", node), extr.config.parser.name("class", node))
+		}
+		textNode := node.Text()
+		ws := extr.config.stopWords.stopWordsCount(extr.config.targetLanguage, textNode)
+		upScore := ws.stopWordCount + int(boostScore)
+		parentNode := node.Parent()
+		extr.updateScore(parentNode, upScore)
+		extr.updateNodeCount(parentNode, 1)
+		if !parentNodes.Has(parentNode) {
+			parentNodes.Add(parentNode)
+		}
+		parentParentNode := parentNode.Parent()
+		if parentParentNode != nil {
+			extr.updateNodeCount(parentParentNode, 1)
+			extr.updateScore(parentParentNode, upScore/2.0)
+			if !parentNodes.Has(parentParentNode) {
+				parentNodes.Add(parentParentNode)
+			}
+		}
+		cnt++
+		i++
+	}
+
+	topNodeScore := 0
+	parentNodesArray := parentNodes.List()
+	for _, p := range parentNodesArray {
+		e := p.(*goquery.Selection)
+		if extr.config.debug {
+			log.Printf("ParentNode: score=%s nodeCount=%s id='%s' class='%s'\n", extr.config.parser.name("gravityScore", e), extr.config.parser.name("gravityNodes", e), extr.config.parser.name("id", e), extr.config.parser.name("class", e))
+		}
+		score := extr.getScore(e)
+		if score >= topNodeScore {
+			topNode = e
+			topNodeScore = score
+		}
+		if topNode == nil {
+			topNode = e
+		}
+	}
+	return topNode
+}
+
+//returns the gravityScore as an integer from this node
+func (extr *ContentExtractor) getScore(node *goquery.Selection) int {
+	return extr.getNodeGravityScore(node)
+}
+
+func (extr *ContentExtractor) getNodeGravityScore(node *goquery.Selection) int {
+	grvScoreString, exists := node.Attr("gravityScore")
+	if !exists {
+		return 0
+	}
+	grvScore, err := strconv.Atoi(grvScoreString)
+	if err != nil {
+		return 0
+	}
+	return grvScore
+}
+
+//adds a score to the gravityScore Attribute we put on divs
+//we'll get the current score then add the score we're passing in to the current
+func (extr *ContentExtractor) updateScore(node *goquery.Selection, addToScore int) {
+	currentScore := 0
+	var err error
+	scoreString, _ := node.Attr("gravityScore")
+	if scoreString != "" {
+		currentScore, err = strconv.Atoi(scoreString)
+		if err != nil {
+			currentScore = 0
+		}
+	}
+	newScore := currentScore + addToScore
+	extr.config.parser.setAttr(node, "gravityScore", strconv.Itoa(newScore))
+}
+
+//stores how many decent nodes are under a parent node
+func (extr *ContentExtractor) updateNodeCount(node *goquery.Selection, addToCount int) {
+	currentScore := 0
+	var err error
+	scoreString, _ := node.Attr("gravityNodes")
+	if scoreString != "" {
+		currentScore, err = strconv.Atoi(scoreString)
+		if err != nil {
+			currentScore = 0
+		}
+	}
+	newScore := currentScore + addToCount
+	extr.config.parser.setAttr(node, "gravityNodes", strconv.Itoa(newScore))
+}
+
+//a lot of times the first paragraph might be the caption under an image so we'll want to make sure if we're going to
+//boost a parent node that it should be connected to other paragraphs, at least for the first n paragraphs
+//so we'll want to make sure that the next sibling is a paragraph and has at least some substantial weight to it
+func (extr *ContentExtractor) isBoostable(node *goquery.Selection) bool {
+	stepsAway := 0
+	next := node.Next()
+	for next != nil && stepsAway < node.Siblings().Length() {
+		currentNodeTag := node.Get(0).DataAtom.String()
+		if currentNodeTag == "p" {
+			if stepsAway >= 3 {
+				if extr.config.debug {
+					log.Println("Next paragraph is too far away, not boosting")
+				}
+				return false
+			}
+
+			paraText := node.Text()
+			ws := extr.config.stopWords.stopWordsCount(extr.config.targetLanguage, paraText)
+			if ws.stopWordCount > 5 {
+				if extr.config.debug {
+					log.Println("We're gonna boost this node, seems content")
+				}
+				return true
+			}
+		}
+
+		stepsAway++
+		next = next.Next()
+	}
+
+	return false
+}
+
+//returns a list of nodes we want to search on like paragraphs and tables
+func (extr *ContentExtractor) nodesToCheck(doc *goquery.Document) []*goquery.Selection {
+	var output []*goquery.Selection
+	tags := []string{"p", "pre", "td"}
+	for _, tag := range tags {
+		selections := doc.Children().Find(tag)
+		if selections != nil {
+			selections.Each(func(i int, s *goquery.Selection) {
+				output = append(output, s)
+			})
+		}
+	}
+	return output
+}
+
+//checks the density of links within a node, is there not much text and most of it contains bad links?
+//if so it's no good
+func (extr *ContentExtractor) isHighLinkDensity(node *goquery.Selection) bool {
+	links := node.Find("a")
+	if links == nil || links.Size() == 0 {
+		return false
+	}
+	text := node.Text()
+	words := strings.Split(text, " ")
+	nwords := len(words)
+	var sb []string
+	links.Each(func(i int, s *goquery.Selection) {
+		linkText := s.Text()
+		sb = append(sb, linkText)
+	})
+	linkText := strings.Join(sb, "")
+	linkWords := strings.Split(linkText, " ")
+	nlinkWords := len(linkWords)
+	nlinks := links.Size()
+	linkDivisor := float64(nlinkWords) / float64(nwords)
+	score := linkDivisor * float64(nlinks)
+
+	if extr.config.debug {
+		var logText string
+		if len(node.Text()) >= 51 {
+			logText = node.Text()[0:50]
+		} else {
+			logText = node.Text()
+		}
+		log.Printf("Calculated link density score as %1.5f for node %s\n", score, logText)
+	}
+	if score > 1.0 {
+		return true
+	}
+	return false
+}
+
+func (extr *ContentExtractor) isTableAndNoParaExist(selection *goquery.Selection) bool {
+	subParagraph := selection.Find("p")
+	subParagraph.Each(func(i int, s *goquery.Selection) {
+		txt := s.Text()
+		if len(txt) < 25 {
+			node := s.Get(0)
+			parent := node.Parent
+			parent.RemoveChild(node)
+		}
+	})
+
+	subParagraph2 := selection.Find("p")
+	if subParagraph2.Length() == 0 && selection.Get(0).DataAtom.String() != "td" {
+		return true
+	}
+	return false
+}
+
+func (extr *ContentExtractor) isNodescoreThresholdMet(node *goquery.Selection, e *goquery.Selection) bool {
+	topNodeScore := extr.getNodeGravityScore(node)
+	currentNodeScore := extr.getNodeGravityScore(e)
+	threasholdScore := float64(topNodeScore) * 0.08
+	if (float64(currentNodeScore) < threasholdScore) && e.Get(0).DataAtom.String() != "td" {
+		return false
+	}
+	return true
+}
+
+//we could have long articles that have tons of paragraphs so if we tried to calculate the base score against
+//the total text score of those paragraphs it would be unfair. So we need to normalize the score based on the average scoring
+//of the paragraphs within the top node. For example if our total score of 10 paragraphs was 1000 but each had an average value of
+//100 then 100 should be our base.
+func (extr *ContentExtractor) getSiblingsScore(topNode *goquery.Selection) int {
+	base := 100000
+	paragraphNumber := 0
+	paragraphScore := 0
+	nodesToCheck := topNode.Find("p")
+	nodesToCheck.Each(func(i int, s *goquery.Selection) {
+		textNode := s.Text()
+		ws := extr.config.stopWords.stopWordsCount(extr.config.targetLanguage, textNode)
+		highLinkDensity := extr.isHighLinkDensity(s)
+		if ws.stopWordCount > 2 && !highLinkDensity {
+			paragraphNumber++
+			paragraphScore += ws.stopWordCount
+		}
+	})
+	if paragraphNumber > 0 {
+		base = paragraphScore / paragraphNumber
+	}
+	return base
+}
+
+func (extr *ContentExtractor) getSiblingsContent(currentSibling *goquery.Selection, baselinescoreSiblingsPara float64) []*goquery.Selection {
+	var ps []*goquery.Selection
+	if currentSibling.Get(0).DataAtom.String() == "p" && len(currentSibling.Text()) > 0 {
+		ps = append(ps, currentSibling)
+		return ps
+	}
+
+	potentialParagraphs := currentSibling.Find("p")
+	potentialParagraphs.Each(func(i int, s *goquery.Selection) {
+		text := s.Text()
+		if len(text) > 0 {
+			ws := extr.config.stopWords.stopWordsCount(extr.config.targetLanguage, text)
+			paragraphScore := ws.stopWordCount
+			siblingBaselineScore := 0.30
+			highLinkDensity := extr.isHighLinkDensity(s)
+			score := siblingBaselineScore * baselinescoreSiblingsPara
+			if score < float64(paragraphScore) && !highLinkDensity {
+				node := new(html.Node)
+				node.Type = html.TextNode
+				node.Data = text
+				node.DataAtom = atom.P
+				nodes := make([]*html.Node, 1)
+				nodes[0] = node
+				newSelection := new(goquery.Selection)
+				newSelection.Nodes = nodes
+				ps = append(ps, newSelection)
+			}
+		}
+
+	})
+	return ps
+}
+
+func (extr *ContentExtractor) walkSiblings(node *goquery.Selection) []*goquery.Selection {
+	currentSibling := node.Prev()
+	var b []*goquery.Selection
+	for currentSibling.Length() != 0 {
+		b = append(b, currentSibling)
+		previousSibling := currentSibling.Prev()
+		currentSibling = previousSibling
+	}
+	return b
+}
+
+//adds any siblings that may have a decent score to this node
+func (extr *ContentExtractor) addSiblings(topNode *goquery.Selection) *goquery.Selection {
+	if extr.config.debug {
+		log.Println("Starting to add siblings")
+	}
+	baselinescoreSiblingsPara := extr.getSiblingsScore(topNode)
+	results := extr.walkSiblings(topNode)
+	for _, currentNode := range results {
+		ps := extr.getSiblingsContent(currentNode, float64(baselinescoreSiblingsPara))
+		for _, p := range ps {
+			nodes := make([]*html.Node, len(topNode.Nodes)+1)
+			nodes[0] = p.Get(0)
+			for i, node := range topNode.Nodes {
+				nodes[i+1] = node
+			}
+			topNode.Nodes = nodes
+		}
+	}
+	return topNode
+}
+
+//PostCleanup removes any divs that looks like non-content, clusters of links, or paras with no gusto
+func (extr *ContentExtractor) PostCleanup(targetNode *goquery.Selection) *goquery.Selection {
+	if extr.config.debug {
+		log.Println("Starting cleanup Node")
+	}
+	node := extr.addSiblings(targetNode)
+	children := node.Children()
+	children.Each(func(i int, s *goquery.Selection) {
+		tag := s.Get(0).DataAtom.String()
+		if tag != "p" {
+			if extr.config.debug {
+				log.Printf("CLEANUP  NODE: %s class: %s\n", extr.config.parser.name("id", s), extr.config.parser.name("class", s))
+			}
+			//if extr.isHighLinkDensity(s) || extr.isTableAndNoParaExist(s) || !extr.isNodescoreThresholdMet(node, s) {
+			if extr.isHighLinkDensity(s) {
+				extr.config.parser.removeNode(s)
+				return
+			}
+
+			subParagraph := s.Find("p")
+			subParagraph.Each(func(j int, e *goquery.Selection) {
+				if len(e.Text()) < 25 {
+					extr.config.parser.removeNode(e)
+				}
+			})
+
+			subParagraph2 := s.Find("p")
+			if subParagraph2.Length() == 0 && tag != "td" {
+				if extr.config.debug {
+					log.Println("Removing node because it doesn't have any paragraphs")
+				}
+				extr.config.parser.removeNode(s)
+			} else {
+				if extr.config.debug {
+					log.Println("Not removing TD node")
+				}
+			}
+			return
+		}
+	})
+	return node
+}
--- a/Показать больше
+++ b/Показать больше