Document extractor service (#15665)

* Document extractor service * Fixing vendor modules * Addressing PR Review comments * Some small simplifications * Fixing a linter complain * simplifying a bit the code using package variables Co-authored-by: Mattermod <mattermod@users.noreply.github.com>
2020-10-27 15:58:38 +01:00
--- a/vendor/github.com/advancedlogic/GoOse/.gitignore
+++ b/vendor/github.com/advancedlogic/GoOse/.gitignore
@@ -0,0 +1,26 @@
+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+*.so
+
+# Folders
+_obj
+_test
+
+# Architecture specific extensions/prefixes
+*.[568vq]
+[568vq].out
+
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+_testmain.go
+
+*.exe
+*.test
+*.prof
+
+target
--- a/vendor/github.com/advancedlogic/GoOse/.travis.yml
+++ b/vendor/github.com/advancedlogic/GoOse/.travis.yml
@@ -0,0 +1,22 @@
+language: go
+
+sudo: false
+
+go:
+  - 1.5
+  - tip
+
+before_install:
+  - go get github.com/mattn/goveralls
+
+before_script:
+  - make deps
+
+script:
+  - make qa
+
+after_failure:
+  - cat ./target/test/report.xml
+
+after_script:
+  - $HOME/gopath/bin/goveralls -covermode=count -coverprofile=target/report/coverage.out -service=travis-ci
--- a/vendor/github.com/advancedlogic/GoOse/Gopkg.lock
+++ b/vendor/github.com/advancedlogic/GoOse/Gopkg.lock
@@ -0,0 +1,129 @@
+# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
+
+
+[[projects]]
+  digest = "1:a62f6ed230a8cd138a9efbe718e7d0b0294f139266f5f55cd942769a9aac8de2"
+  name = "github.com/PuerkitoBio/goquery"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "dc2ec5c7ca4d9aae063b79b9f581dd3ea6afd2b2"
+  version = "v1.4.1"
+
+[[projects]]
+  digest = "1:66b3310cf22cdc96c35ef84ede4f7b9b370971c4025f394c89a2638729653b11"
+  name = "github.com/andybalholm/cascadia"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "901648c87902174f774fac311d7f176f8647bdaa"
+  version = "v1.0.0"
+
+[[projects]]
+  branch = "master"
+  digest = "1:90d593cd34221cf33017a2c944998eb29f5182a15cfbe0cf6e2a0f62cb8bbf29"
+  name = "github.com/araddon/dateparse"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "cfd92a431d0efe36a1b81ca25d15b98aae4dbdb6"
+
+[[projects]]
+  digest = "1:5f1d44e13c151d777a80b0d8ca850347530773e73ccce5995a520cdc6947c205"
+  name = "github.com/fatih/set"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "2c768e3c5489976167bfc42b5c7c92ca783f4389"
+  version = "v0.2.1"
+
+[[projects]]
+  branch = "master"
+  digest = "1:131e93e8b8f318c52b1666e5a8f3af7d236af62f5a477a11973e46af59bca7da"
+  name = "github.com/gigawattio/window"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "0f5467e35573c4ffebddd992935e1f7ceeac0696"
+
+[[projects]]
+  branch = "master"
+  digest = "1:8bacc2118e6cc6a207eb144656537ff7ee0397f8bc8c1f7639519b0de2bb949a"
+  name = "github.com/jaytaylor/html2text"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "57d518f124b0cf46ea2021f25a01396b3522e6fb"
+
+[[projects]]
+  digest = "1:cdb899c199f907ac9fb50495ec71212c95cb5b0e0a8ee0800da0238036091033"
+  name = "github.com/mattn/go-runewidth"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "ce7b0b5c7b45a81508558cd1dba6bb1e4ddb51bb"
+  version = "v0.0.3"
+
+[[projects]]
+  branch = "master"
+  digest = "1:4daa045e1e1f3e23f4b07db6880cdf9f259dab65312dfe244a878e6070faaf77"
+  name = "github.com/olekukonko/tablewriter"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "d4647c9c7a84d847478d890b816b7d8b62b0b279"
+
+[[projects]]
+  branch = "master"
+  digest = "1:362b269942e8508aed33025593090ad4cdbd9077de4b1cd505e6124728e9e856"
+  name = "github.com/ssor/bom"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "6386211fdfcf24c0bfbdaceafd02849ed9a8a509"
+
+[[projects]]
+  branch = "master"
+  digest = "1:1d3f083a0b29a9db5ac57dc4d486dbb97cd58696bae3da441a375916b7e8c53b"
+  name = "golang.org/x/net"
+  packages = [
+    "html",
+    "html/atom",
+    "html/charset",
+  ]
+  pruneopts = "UT"
+  revision = "4bcd98cce591d8c7061bf313d7a3cbad05b58549"
+
+[[projects]]
+  digest = "1:aa4d6967a3237f8367b6bf91503964a77183ecf696f1273e8ad3551bb4412b5f"
+  name = "golang.org/x/text"
+  packages = [
+    "encoding",
+    "encoding/charmap",
+    "encoding/htmlindex",
+    "encoding/internal",
+    "encoding/internal/identifier",
+    "encoding/japanese",
+    "encoding/korean",
+    "encoding/simplifiedchinese",
+    "encoding/traditionalchinese",
+    "encoding/unicode",
+    "internal/gen",
+    "internal/tag",
+    "internal/utf8internal",
+    "language",
+    "runes",
+    "transform",
+    "unicode/cldr",
+  ]
+  pruneopts = "UT"
+  revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
+  version = "v0.3.0"
+
+[solve-meta]
+  analyzer-name = "dep"
+  analyzer-version = 1
+  input-imports = [
+    "github.com/PuerkitoBio/goquery",
+    "github.com/araddon/dateparse",
+    "github.com/fatih/set",
+    "github.com/gigawattio/window",
+    "github.com/jaytaylor/html2text",
+    "golang.org/x/net/html",
+    "golang.org/x/net/html/atom",
+    "golang.org/x/net/html/charset",
+    "golang.org/x/text/transform",
+  ]
+  solver-name = "gps-cdcl"
+  solver-version = 1
--- a/vendor/github.com/advancedlogic/GoOse/Gopkg.toml
+++ b/vendor/github.com/advancedlogic/GoOse/Gopkg.toml
@@ -0,0 +1,58 @@
+# Gopkg.toml example
+#
+# Refer to https://golang.github.io/dep/docs/Gopkg.toml.html
+# for detailed Gopkg.toml documentation.
+#
+# required = ["github.com/user/thing/cmd/thing"]
+# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
+#
+# [[constraint]]
+#   name = "github.com/user/project"
+#   version = "1.0.0"
+#
+# [[constraint]]
+#   name = "github.com/user/project2"
+#   branch = "dev"
+#   source = "github.com/myfork/project2"
+#
+# [[override]]
+#   name = "github.com/x/y"
+#   version = "2.4.0"
+#
+# [prune]
+#   non-go = false
+#   go-tests = true
+#   unused-packages = true
+
+
+[[constraint]]
+  name = "github.com/PuerkitoBio/goquery"
+  version = "1.4.1"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/araddon/dateparse"
+
+[[constraint]]
+  name = "github.com/fatih/set"
+  version = "0.2.1"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/gigawattio/window"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/jaytaylor/html2text"
+
+[[constraint]]
+  branch = "master"
+  name = "golang.org/x/net"
+
+[[constraint]]
+  name = "golang.org/x/text"
+  version = "0.3.0"
+
+[prune]
+  go-tests = true
+  unused-packages = true
--- a/vendor/github.com/advancedlogic/GoOse/LICENSE
+++ b/vendor/github.com/advancedlogic/GoOse/LICENSE
@@ -0,0 +1,202 @@
+Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
--- a/vendor/github.com/advancedlogic/GoOse/Makefile
+++ b/vendor/github.com/advancedlogic/GoOse/Makefile
@@ -0,0 +1,203 @@
+# MAKEFILE
+#
+# @author      Nicola Asuni <info@tecnick.com>
+# @link        https://github.com/advancedlogic/GoOse
+#
+# This file is intended to be executed in a Linux-compatible system.
+# It also assumes that the project has been cloned in the right path under GOPATH:
+# $GOPATH/src/github.com/advancedlogic/GoOse
+#
+# ------------------------------------------------------------------------------
+
+# List special make targets that are not associated with files
+.PHONY: help all test format fmtcheck vet lint coverage cyclo ineffassign misspell structcheck varcheck errcheck gosimple astscan qa deps clean nuke
+
+# Use bash as shell (Note: Ubuntu now uses dash which doesn't support PIPESTATUS).
+SHELL=/bin/bash
+
+# CVS path (path to the parent dir containing the project)
+CVSPATH=github.com/advancedlogic
+
+# Project owner
+OWNER=advancedlogic
+
+# Project vendor
+VENDOR=advancedlogic
+
+# Project name
+PROJECT=GoOse
+
+# Project version
+VERSION=$(shell cat VERSION)
+
+# Name of RPM or DEB package
+PKGNAME=${VENDOR}-${PROJECT}
+
+# Current directory
+CURRENTDIR=$(shell pwd)
+
+# GO lang path
+ifneq ($(GOPATH),)
+	ifeq ($(findstring $(GOPATH),$(CURRENTDIR)),)
+		# the defined GOPATH is not valid
+		GOPATH=
+	endif
+endif
+ifeq ($(GOPATH),)
+	# extract the GOPATH
+	GOPATH=$(firstword $(subst /src/, ,$(CURRENTDIR)))
+endif
+
+# --- MAKE TARGETS ---
+
+# Display general help about this command
+help:
+	@echo ""
+	@echo "$(PROJECT) Makefile."
+	@echo "GOPATH=$(GOPATH)"
+	@echo "The following commands are available:"
+	@echo ""
+	@echo "    make qa          : Run all the tests"
+	@echo "    make test        : Run the unit tests"
+	@echo ""
+	@echo "    make format      : Format the source code"
+	@echo "    make fmtcheck    : Check if the source code has been formatted"
+	@echo "    make vet         : Check for suspicious constructs"
+	@echo "    make lint        : Check for style errors"
+	@echo "    make coverage    : Generate the coverage report"
+	@echo "    make cyclo       : Generate the cyclomatic complexity report"
+	@echo "    make ineffassign : Detect ineffectual assignments"
+	@echo "    make misspell    : Detect commonly misspelled words in source files"
+	@echo "    make structcheck : Find unused struct fields"
+	@echo "    make varcheck    : Find unused global variables and constants"
+	@echo "    make errcheck    : Check that error return values are used"
+	@echo "    make gosimple    : Suggest code simplifications"
+	@echo "    make astscan     : GO AST scanner"
+	@echo ""
+	@echo "    make docs        : Generate source code documentation"
+	@echo ""
+	@echo "    make deps        : Get the dependencies"
+	@echo "    make clean       : Remove any build artifact"
+	@echo "    make nuke        : Deletes any intermediate file"
+	@echo ""
+
+
+# Alias for help target
+all: help
+
+# Run the unit tests
+test:
+	@mkdir -p target/test
+	@mkdir -p target/report
+	GOPATH=$(GOPATH) \
+	go test \
+	-covermode=atomic \
+	-bench=. \
+	-race \
+	-cpuprofile=target/report/cpu.out \
+	-memprofile=target/report/mem.out \
+	-mutexprofile=target/report/mutex.out \
+	-coverprofile=target/report/coverage.out \
+	-v ./... | \
+	tee >(PATH=$(GOPATH)/bin:$(PATH) go-junit-report > target/test/report.xml); \
+	test $${PIPESTATUS[0]} -eq 0
+
+# Format the source code
+format:
+	@find . -type f -name "*.go" -exec gofmt -s -w {} \;
+
+# Check if the source code has been formatted
+fmtcheck:
+	@mkdir -p target
+	@find . -type f -name "*.go" -exec gofmt -s -d {} \; | tee target/format.diff
+	@test ! -s target/format.diff || { echo "ERROR: the source code has not been formatted - please use 'make format' or 'gofmt'"; exit 1; }
+
+# Check for syntax errors
+vet:
+	GOPATH=$(GOPATH) go vet .
+
+# Check for style errors
+lint:
+	GOPATH=$(GOPATH) PATH=$(GOPATH)/bin:$(PATH) golint .
+
+# Generate the coverage report
+coverage:
+	@mkdir -p target/report
+	GOPATH=$(GOPATH) \
+	go tool cover -html=target/report/coverage.out -o target/report/coverage.html
+
+# Report cyclomatic complexity
+cyclo:
+	@mkdir -p target/report
+	GOPATH=$(GOPATH) gocyclo -avg ./ | tee target/report/cyclo.txt ; test $${PIPESTATUS[0]} -eq 0
+
+# Detect ineffectual assignments
+ineffassign:
+	@mkdir -p target/report
+	GOPATH=$(GOPATH) ineffassign ./ | tee target/report/ineffassign.txt ; test $${PIPESTATUS[0]} -eq 0
+
+# Detect commonly misspelled words in source files
+misspell:
+	@mkdir -p target/report
+	GOPATH=$(GOPATH) misspell -error ./*.go | tee target/report/misspell.txt
+
+# Find unused struct fields
+structcheck:
+	@mkdir -p target/report
+	GOPATH=$(GOPATH) structcheck -a ./ | tee target/report/structcheck.txt
+
+# Find unused global variables and constants
+varcheck:
+	@mkdir -p target/report
+	GOPATH=$(GOPATH) varcheck -e ./ | tee target/report/varcheck.txt
+
+# Check that error return values are used
+errcheck:
+	@mkdir -p target/report
+	GOPATH=$(GOPATH) errcheck ./ | tee target/report/errcheck.txt
+
+# Suggest code simplifications
+gosimple:
+	@mkdir -p target/report
+	GOPATH=$(GOPATH) gosimple ./ | tee target/report/gosimple.txt
+
+# AST scanner
+astscan:
+	@mkdir -p target/report
+	GOPATH=$(GOPATH) gas .//*.go | tee target/report/astscan.txt
+
+# Generate source docs
+docs:
+	@mkdir -p target/docs
+	nohup sh -c 'GOPATH=$(GOPATH) godoc -http=127.0.0.1:6060' > target/godoc_server.log 2>&1 &
+	wget --directory-prefix=target/docs/ --execute robots=off --retry-connrefused --recursive --no-parent --adjust-extension --page-requisites --convert-links http://127.0.0.1:6060/pkg/github.com/${VENDOR}/${PROJECT}/ ; kill -9 `lsof -ti :6060`
+	@echo '<html><head><meta http-equiv="refresh" content="0;./127.0.0.1:6060/pkg/'${CVSPATH}'/'${PROJECT}'/index.html"/></head><a href="./127.0.0.1:6060/pkg/'${CVSPATH}'/'${PROJECT}'/index.html">'${PKGNAME}' Documentation ...</a></html>' > target/docs/index.html
+
+# Alias to run all quality-assurance checks
+qa: fmtcheck test vet lint coverage cyclo ineffassign misspell structcheck varcheck errcheck gosimple astscan
+
+# --- INSTALL ---
+
+# Get the dependencies
+deps:
+	GOPATH=$(GOPATH) go get ./...
+	GOPATH=$(GOPATH) go get github.com/golang/lint/golint
+	GOPATH=$(GOPATH) go get github.com/jstemmer/go-junit-report
+	GOPATH=$(GOPATH) go get github.com/axw/gocov/gocov
+	GOPATH=$(GOPATH) go get github.com/fzipp/gocyclo
+	GOPATH=$(GOPATH) go get github.com/gordonklaus/ineffassign
+	GOPATH=$(GOPATH) go get github.com/client9/misspell/cmd/misspell
+	GOPATH=$(GOPATH) go get github.com/opennota/check/cmd/structcheck
+	GOPATH=$(GOPATH) go get github.com/opennota/check/cmd/varcheck
+	GOPATH=$(GOPATH) go get github.com/kisielk/errcheck
+	GOPATH=$(GOPATH) go get honnef.co/go/tools/cmd/gosimple
+	GOPATH=$(GOPATH) go get github.com/GoASTScanner/gas
+
+# Remove any build artifact
+clean:
+	GOPATH=$(GOPATH) go clean ./...
+
+# Deletes any intermediate file
+nuke:
+	rm -rf ./target
+	GOPATH=$(GOPATH) go clean -i ./...
--- a/vendor/github.com/advancedlogic/GoOse/README.md
+++ b/vendor/github.com/advancedlogic/GoOse/README.md
@@ -0,0 +1,86 @@
+# GoOse
+
+*HTML Content / Article Extractor in Golang*
+
+[![Build Status](https://secure.travis-ci.org/advancedlogic/GoOse.png?branch=master)](https://travis-ci.org/advancedlogic/GoOse?branch=master)
+[![Coverage Status](https://coveralls.io/repos/advancedlogic/GoOse/badge.svg?branch=master&service=github)](https://coveralls.io/github/advancedlogic/GoOse?branch=master)
+[![Go Report Card](https://goreportcard.com/badge/github.com/advancedlogic/GoOse)](https://goreportcard.com/report/github.com/advancedlogic/GoOse)
+[![GoDoc](https://godoc.org/github.com/advancedlogic/GoOse?status.svg)](http://godoc.org/github.com/advancedlogic/GoOse)
+
+
+## Description
+
+This is a golang port of "Goose" originaly licensed to Gravity.com
+under one or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.
+
+Golang port was written by Antonio Linari
+
+Gravity.com licenses this file
+to you under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+## INSTALL
+
+```bash
+go get github.com/advancedlogic/GoOse
+```
+
+## HOW TO USE IT
+
+```Go
+package main
+
+import (
+	"github.com/advancedlogic/GoOse"
+)
+
+func main() {
+	g := goose.New()
+	article, _ := g.ExtractFromURL("http://edition.cnn.com/2012/07/08/opinion/banzi-ted-open-source/index.html")
+	println("title", article.Title)
+	println("description", article.MetaDescription)
+	println("keywords", article.MetaKeywords)
+	println("content", article.CleanedText)
+	println("url", article.FinalURL)
+	println("top image", article.TopImage)
+}
+```
+
+## Development - Getting started
+
+This application is written in GO language, please refere to the guides in https://golang.org for getting started.
+
+This project include a Makefile that allows you to test and build the project with simple commands.
+To see all available options:
+```bash
+make help
+```
+
+Before committing the code, please check if it passes all tests using
+```bash
+make deps
+make qa
+```
+
+## TODO
+- [ ] better organize code
+- [ ] improve "xpath" like queries
+- [ ] add other image extractions techniques (imagemagick)
+
+## THANKS TO
+```
+@Martin Angers for goquery
+@Fatih Arslan for set
+GoLang team for the amazing language and net/html
+```
--- a/vendor/github.com/advancedlogic/GoOse/VERSION
+++ b/vendor/github.com/advancedlogic/GoOse/VERSION
@@ -0,0 +1 @@
+1.1.0
--- a/vendor/github.com/advancedlogic/GoOse/article.go
+++ b/vendor/github.com/advancedlogic/GoOse/article.go
@@ -0,0 +1,40 @@
+package goose
+
+import (
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/fatih/set"
+)
+
+// Article is a collection of properties extracted from the HTML body
+type Article struct {
+	Title           string             `json:"title,omitempty"`
+	TitleUnmodified string             `json:"titleunmodified,omitempty"`
+	CleanedText     string             `json:"content,omitempty"`
+	MetaDescription string             `json:"description,omitempty"`
+	MetaLang        string             `json:"lang,omitempty"`
+	MetaFavicon     string             `json:"favicon,omitempty"`
+	MetaKeywords    string             `json:"keywords,omitempty"`
+	CanonicalLink   string             `json:"canonicalurl,omitempty"`
+	Domain          string             `json:"domain,omitempty"`
+	TopNode         *goquery.Selection `json:"-"`
+	TopImage        string             `json:"image,omitempty"`
+	Tags            *set.Set           `json:"tags,omitempty"`
+	Movies          *set.Set           `json:"movies,omitempty"`
+	FinalURL        string             `json:"url,omitempty"`
+	LinkHash        string             `json:"linkhash,omitempty"`
+	RawHTML         string             `json:"rawhtml,omitempty"`
+	Doc             *goquery.Document  `json:"-"`
+	Links           []string           `json:"links,omitempty"`
+	PublishDate     *time.Time         `json:"publishdate,omitempty"`
+	AdditionalData  map[string]string  `json:"additionaldata,omitempty"`
+	Delta           int64              `json:"delta,omitempty"`
+}
+
+// ToString is a simple method to just show the title
+// TODO: add more fields and pretty print
+func (article *Article) ToString() string {
+	out := article.Title
+	return out
+}
--- a/vendor/github.com/advancedlogic/GoOse/charset.go
+++ b/vendor/github.com/advancedlogic/GoOse/charset.go
@@ -0,0 +1,92 @@
+package goose
+
+import (
+	"log"
+	"strings"
+	"unicode/utf8"
+
+	"golang.org/x/net/html/charset"
+	"golang.org/x/text/transform"
+)
+
+// NormaliseCharset Overrides/fixes charset names to something we can parse.
+// Fixes common mispellings and uses a canonical name for equivalent encodings.
+// @see https://encoding.spec.whatwg.org#names-and-labels
+func NormaliseCharset(characterSet string) string {
+	characterSet = strings.ToUpper(characterSet)
+	switch characterSet {
+	case "UTF8", "UT-8", "UTR-8", "UFT-8", "UTF8-WITHOUT-BOM", "UTF8_GENERAL_CI":
+		return "UTF-8"
+	// override Japanese
+	// CP943: IBM OS/2 Japanese, superset of Cp932 and Shift-JIS
+	case "CP943", "CP943C", "SIFT_JIS", "SHIFT-JIS":
+		return "SHIFT_JIS"
+	// override Korean
+	case "EUC-KR", "MS949", "KSC5601", "WINDOWS-949", "KS_C_5601-1987", "KSC_5601":
+		return "UHC"
+	// override Thai
+	//case "TIS-620", "WINDOWS-874":
+	//	return "ISO-8859-11"
+	// override latin-2
+	case "LATIN2_HUNGARIAN_CI", "LATIN2":
+		return "LATIN-2"
+	// override cyrillic
+	case "WIN1251", "WIN-1251", "WINDOWS-1251":
+		return "CP1251"
+	// override Hebrew
+	case "WINDOWS-1255":
+		return "ISO-8859-8"
+	// override Turkish
+	//case "WINDOWS-1254":
+	//	return "ISO-8859-9"
+	// override the parsing of ISO-8859-1 to behave as Windows-1252 (CP1252):
+	// in ISO-8859-1, everything from 128-255 in the ASCII table are ctrl characters,
+	// whilst in CP1252 they're symbols
+	// override Baltic
+	case "WINDOWS-1257":
+		return "ISO-8859-13"
+	case "ANSI", "LATIN-1", "ISO", "RFC", "MACINTOSH", "8859-1", "8859-15", "ISO8859-1", "ISO8859-15", "ISO-8559-1", "ISO-8859-1", "ISO-8859-15":
+		return "CP1252"
+	}
+	return characterSet
+}
+
+// UTF8encode converts a string from the source character set to UTF-8, skipping invalid byte sequences
+// @see http://stackoverflow.com/questions/32512500/ignore-illegal-bytes-when-decoding-text-with-go
+func UTF8encode(raw string, sourceCharset string) string {
+	enc, name := charset.Lookup(sourceCharset)
+	if nil == enc {
+		log.Println("Cannot convert from", sourceCharset, ":", name)
+		return raw
+	}
+
+	dst := make([]byte, len(raw))
+	d := enc.NewDecoder()
+
+	var (
+		in  int
+		out int
+	)
+	for in < len(raw) {
+		// Do the transformation
+		ndst, nsrc, err := d.Transform(dst[out:], []byte(raw[in:]), true)
+		in += nsrc
+		out += ndst
+		if err == nil {
+			// Completed transformation
+			break
+		}
+		if err == transform.ErrShortDst {
+			// Our output buffer is too small, so we need to grow it
+			t := make([]byte, (cap(dst)+1)*2)
+			copy(t, dst)
+			dst = t
+			continue
+		}
+		// We're here because of at least one illegal character. Skip over the current rune
+		// and try again.
+		_, width := utf8.DecodeRuneInString(raw[in:])
+		in += width
+	}
+	return string(dst)
+}
--- a/vendor/github.com/advancedlogic/GoOse/cleaner.go
+++ b/vendor/github.com/advancedlogic/GoOse/cleaner.go
@@ -0,0 +1,568 @@
+package goose
+
+import (
+	"container/list"
+	"github.com/PuerkitoBio/goquery"
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/atom"
+	"log"
+	"regexp"
+	"strings"
+)
+
+var whitelistedTextAtomTypes = []atom.Atom{atom.Span, atom.Em, atom.I, atom.Strong, atom.B, atom.P, atom.H1, atom.H2, atom.H3, atom.H4}
+var whitelistedExtAtomTypes = []atom.Atom{atom.A, atom.Span, atom.Em, atom.I, atom.Strong, atom.B, atom.P, atom.H1, atom.H2, atom.H3, atom.H4}
+
+// Cleaner removes menus, ads, sidebars, etc. and leaves the main content
+type Cleaner struct {
+	config Configuration
+}
+
+// NewCleaner returns a new instance of a Cleaner
+func NewCleaner(config Configuration) Cleaner {
+	return Cleaner{
+		config: config,
+	}
+}
+
+// replaceTagWithContents removes the tag, replacing it with its text contents
+// e.g. "<em>some text</em>" becomes "some text"
+func replaceTagWithContents(tagSelection *goquery.Selection, collapsibleAtomTypes []atom.Atom) {
+	if tagSelection.Length() == 0 {
+		return
+	}
+	node := tagSelection.Get(0)
+	node.Data = tagSelection.Text()
+	node.Type = html.TextNode
+	if node.FirstChild == nil {
+		node.Attr = []html.Attribute{}
+		node.DataAtom = 0
+		node.FirstChild = nil
+		node.LastChild = nil
+	} else {
+		// If all children are text only, the parent already contains the text, so drop them
+		collapseTextNodes(node, collapsibleAtomTypes)
+	}
+}
+
+func isAtomTypeWhitelisted(t atom.Atom, whitelist []atom.Atom) bool {
+	for _, allowed := range whitelist {
+		if t == allowed {
+			return true
+		}
+	}
+	return false
+}
+
+func collapseTextNodes(node *html.Node, collapsibleAtomTypes []atom.Atom) {
+	if node.FirstChild == nil {
+		return
+	}
+
+	if !isAtomTypeWhitelisted(node.DataAtom, collapsibleAtomTypes) {
+		return
+	}
+
+	if node.FirstChild.DataAtom == 0 && node.FirstChild == node.LastChild {
+		// this tag only contains a single textual node, already contained in the parent
+		node.Attr = []html.Attribute{}
+		node.Type = html.TextNode
+		node.DataAtom = 0
+		node.FirstChild = nil
+		node.LastChild = nil
+		return
+	}
+
+	// If all children are text only, the parent already contains the text, so drop them
+	allTextNodes := true
+	for c := node.FirstChild; c != nil; c = c.NextSibling {
+		// attempt collapsing recursively
+		collapseTextNodes(c, collapsibleAtomTypes)
+		if c.DataAtom != 0 {
+			// not collapsed
+			allTextNodes = false
+			break
+		}
+	}
+	if allTextNodes {
+		// text already contained in the parent node => drop children
+		node.Attr = []html.Attribute{}
+		node.Type = html.TextNode
+		node.DataAtom = 0
+		node.FirstChild = nil
+		node.LastChild = nil
+	}
+}
+
+var divToPElementsPattern = regexp.MustCompile("<(a|blockquote|dl|div|img|ol|p|pre|table|ul)")
+var tabsRegEx = regexp.MustCompile(`\t|^\s+$]`)
+var removeVisibilityStyleRegEx = regexp.MustCompile("visibility:[ ]*hidden|display:[ ]*none")
+var removeNodesRegEx = regexp.MustCompile("" +
+	"[Cc]omentario|" +
+	"[Ff]ooter|" +
+	"^fn$|" +
+	"^inset$|" +
+	"^print$|" +
+	"^scroll$|" +
+	"^side$|" +
+	"^side_|" +
+	"^widget$|" +
+	"^ab[0-9]$|" +
+	"[_-]ads$|" +
+	"^ad[s]?[ _-]|" +
+	"[_-]ad[s]?[_-]|" +
+	"^ADX_CLIENTSIDE$|" +
+	"ajoutVideo|" +
+	"^alerts|" +
+	"^Anchor$|" +
+	"articleheadings|" +
+	"_articles|" +
+	"^article-gallery-embedded$|" +
+	"author|" +
+	"author-dropdown|" +
+	"^banner|" +
+	"^bar$|" +
+	"blog-pager|" +
+	"brass\\-rail|" +
+	"breadcrumbs|" +
+	"button|" +
+	"byline|" +
+	"cabecalho|" +
+	"^caption$|" +
+	"carousel|" +
+	"^click|" +
+	"cnnStryHghLght|" +
+	"cnn_html_slideshow|" +
+	"cnn_strycaptiontxt|" +
+	"cnn_strylftcntnt|" +
+	"cnn_stryspcvbx|" +
+	"combx|" +
+	"comment|" +
+	"commercial|" +
+	"communitypromo|" +
+	"^comscore$|" +
+	"contact|" +
+	"contentTools2|" +
+	"controls|" +
+	"cookie|" +
+	"CoversMainContent|" +
+	"^css-|" +
+	"^critical-alerts$|" +
+	"^date$|" +
+	"detail_new_|" +
+	"downloadLink|" +
+	"^DYSRC$|" +
+	"^early-body|" +
+	"ec_blogs|" +
+	"^[^entry-]more.*$|" +
+	"error|" +
+	"[^-]facebook|" +
+	"facebook-broadcasting|" +
+	"^fb-root$|" +
+	"^feed[_-]|" +
+	"figcaption|" +
+	"footnote|" +
+	"foot|" +
+	"footer|" +
+	"^ga-track$|" +
+	" google |" +
+	"^gstl_|" +
+	"^GS-UH$|" +
+	"^guide$|" +
+	"header|" +
+	"hidden|" +
+	"img_popup_single|" +
+	"inline-share-tools|" +
+	"inread|" +
+	"^interstitial-ad-modal$|" +
+	"^Inv[0-9]$|" +
+	"js_replies|" +
+	"[Kk]ona[Ff]ilter|" +
+	"^kxhead$|" +
+	"leading|" +
+	"^lede[_-]container$|" +
+	"legende?|" +
+	"^lightningjs-|" +
+	"links|" +
+	"^login-modal$|" +
+	"^lui-mini-profile-body$|" +
+	"^marginalia|" +
+	"^marketing[_-]|" +
+	"^masthead|" +
+	"mediaarticlerelated|" +
+	"^media[_-]viewer$|" +
+	"menu|" +
+	"menucontainer|" +
+	"meta$|" +
+	"^moat$|" +
+	"moreNews|" +
+	"^Moses$|" +
+	"^nav[_-]|" +
+	"navbar|" +
+	"[Nn]avigation|" +
+	"newsUnder|" +
+	"^oauth|" +
+	"^overlay[_-]wrapper|" +
+	"pagetools|" +
+	"[_-]paid[_-]|" +
+	"panelss2|" +
+	"panesCity|" +
+	"player|" +
+	"PopularQuestions|" +
+	"popup|" +
+	"post[_-]attributes|" +
+	"post[_-]title|" +
+	"preview|" +
+	"[_-]print[_-]|" +
+	"products\\-events|" +
+	"^prop[0-9]$|" +
+	"^pulse-loaders|" +
+	"^rail$|" +
+	"recommend|" +
+	"^registration-modal$|" +
+	"relacionado|" +
+	"related|" +
+	"remote|" +
+	"retweet|" +
+	"^ribbon$|" +
+	"rightBlock|" +
+	"rss|" +
+	"runaroundLeft|" +
+	"search[_-]|" +
+	"share[_-]|" +
+	"shoutbox|" +
+	"sidebar|" +
+	"^simplereach$|" +
+	"^site[_-]index$|" +
+	"site[_-]box|" +
+	"site[_-]nav|" +
+	"skyscraper|" +
+	"social[Nn]etworking|" +
+	"social_|" +
+	"social\\-share|" +
+	"social\\-count|" +
+	"socialtools|" +
+	"source|" +
+	"^speed-bump-wrapper$|" +
+	"[_-]spinner$|" +
+	"^Splash$|" +
+	"sponsor|" +
+	"^spr-|" +
+	"storytopbar\\-bucket|" +
+	"^stream-sidebar|" +
+	"sub_nav|" +
+	"subscribe|" +
+	"subscription|" +
+	"^suggestions$|" +
+	"tabsCity|" +
+	"tag_|" +
+	"tags|" +
+	"teaser|" +
+	"the_answers|" +
+	"timestamp|" +
+	"tools|" +
+	"tooltip|" +
+	"^Top[0-9]?$|" +
+	"^TopAd[0-9]?$|" +
+	"[_-]track[_-]|" +
+	"tracking|" +
+	"[^-]twitter|" +
+	"-uix-button|" +
+	"updateBrowser|" +
+	"^username-modal$|" +
+	"^user-|" +
+	"utility-bar|" +
+	"^vestpocket$|" +
+	"vcard|" +
+	"^watch-action-panels$|" +
+	"^watch-discussion$|" +
+	"welcome_form|" +
+	"^whats[_-]next$|" +
+	"wp-caption-text")
+
+// Clean removes HTML elements around the main content and prepares the document for parsing
+func (c *Cleaner) Clean(docToClean *goquery.Document) *goquery.Document {
+	if c.config.debug {
+		log.Println("Starting cleaning phase with Cleaner")
+	}
+	docToClean = c.cleanBr(docToClean)
+	docToClean = c.cleanArticleTags(docToClean)
+	docToClean = c.cleanEMTags(docToClean)
+	docToClean = c.dropCaps(docToClean)
+	docToClean = c.removeScriptsStyle(docToClean)
+	docToClean = c.cleanBadTags(docToClean, removeNodesRegEx, &[]string{"id", "class", "name"})
+	docToClean = c.cleanBadTags(docToClean, removeVisibilityStyleRegEx, &[]string{"style"})
+	docToClean = c.removeTags(docToClean, &[]string{"nav", "footer", "aside", "cite"})
+	docToClean = c.cleanParaSpans(docToClean)
+
+
+	docToClean = c.convertDivsToParagraphs(docToClean, "div")
+
+	docToClean = c.convertDivsToParagraphs(docToClean, "span")
+	docToClean = c.convertDivsToParagraphs(docToClean, "article")
+	docToClean = c.convertDivsToParagraphs(docToClean, "pre")
+
+	return docToClean
+}
+
+func (c *Cleaner) cleanArticleTags(doc *goquery.Document) *goquery.Document {
+	tags := [3]string{"id", "name", "class"}
+	articles := doc.Find("article")
+	articles.Each(func(i int, s *goquery.Selection) {
+		for _, tag := range tags {
+			c.config.parser.delAttr(s, tag)
+		}
+	})
+	return doc
+}
+
+// replace <br /> with \n\n
+func (c *Cleaner) cleanBr(doc *goquery.Document) *goquery.Document {
+	linebreaks := doc.Find("br")
+	linebreaks.Each(func(i int, br *goquery.Selection) {
+		node := br.Get(0)
+		node.Data = "\n\n"
+		node.Type = html.TextNode
+		node.Attr = []html.Attribute{}
+		node.DataAtom = 0
+		node.FirstChild = nil
+		node.LastChild = nil
+	})
+	return doc
+}
+
+func (c *Cleaner) cleanEMTags(doc *goquery.Document) *goquery.Document {
+	ems := doc.Find("em")
+	ems.Each(func(i int, s *goquery.Selection) {
+		images := s.Find("img")
+		if images.Length() == 0 {
+			c.config.parser.dropTag(s)
+		}
+	})
+	if c.config.debug {
+		log.Printf("Cleaning %d EM tags\n", ems.Size())
+	}
+	return doc
+}
+
+func (c *Cleaner) removeTags(doc *goquery.Document, tags *[]string) *goquery.Document {
+	for _, tag := range *tags {
+		node := doc.Find(tag)
+		node.Each(func(i int, s *goquery.Selection) {
+			c.config.parser.removeNode(s)
+		})
+	}
+	return doc
+}
+
+func (c *Cleaner) cleanDivs(doc *goquery.Document) *goquery.Document {
+	frames := make(map[string]int)
+	framesNodes := make(map[string]*list.List)
+	divs := doc.Find("div")
+	divs.Each(func(i int, s *goquery.Selection) {
+		children := s.Children()
+		if children.Size() == 0 {
+			text := strings.Trim(s.Text(), " \t")
+			text = strings.ToLower(text)
+			frames[text]++
+			if framesNodes[text] == nil {
+				framesNodes[text] = list.New()
+			}
+			framesNodes[text].PushBack(s)
+		}
+	})
+	for text, freq := range frames {
+		if freq > 1 {
+			selections := framesNodes[text]
+			for s := selections.Front(); s != nil; s = s.Next() {
+				selection := s.Value.(*goquery.Selection)
+				c.config.parser.removeNode(selection)
+			}
+		}
+	}
+	return doc
+}
+
+func (c *Cleaner) dropCaps(doc *goquery.Document) *goquery.Document {
+	items := doc.Find("span")
+	count := 0 // remove
+	items.Each(func(i int, s *goquery.Selection) {
+		attribute, exists := s.Attr("class")
+		if exists && (strings.Contains(attribute, "dropcap") || strings.Contains(attribute, "drop_cap")) {
+			c.config.parser.dropTag(s)
+			count++
+		}
+	})
+	if c.config.debug && count > 0 {
+		log.Printf("Cleaned %d dropcap tags\n", count)
+	}
+	return doc
+}
+
+func (c *Cleaner) removeScriptsStyle(doc *goquery.Document) *goquery.Document {
+	if c.config.debug {
+		log.Println("Starting to remove script tags")
+	}
+	count := 0 // number of removed nodes
+	scripts := doc.Find("script,noscript,style")
+	scripts.Each(func(i int, s *goquery.Selection) {
+		c.config.parser.removeNode(s)
+		count++
+	})
+	if c.config.debug && count > 0 {
+		log.Printf("Removed %d script and style tags\n", scripts.Size())
+	}
+	return doc
+}
+
+func (c *Cleaner) cleanBadTags(doc *goquery.Document, pattern *regexp.Regexp, selectors *[]string) *goquery.Document {
+	body := doc.Find("html")
+	children := body.Children()
+	children.Each(func(i int, s *goquery.Selection) {
+		for _, selector := range *selectors {
+			naughtyList := s.Find("*[" + selector + "]")
+			count := 0
+			naughtyList.Each(func(j int, node *goquery.Selection) {
+				attribute, _ := node.Attr(selector)
+				if pattern.MatchString(attribute) {
+					if c.config.debug {
+						log.Printf("Cleaning: Removing node with %s: %s\n", selector, c.config.parser.name(selector, node))
+					}
+					c.config.parser.removeNode(node)
+					count++
+				}
+			})
+			if c.config.debug && count > 0 {
+				log.Printf("%d naughty %s elements found", count, selector)
+			}
+		}
+	})
+	return doc
+}
+
+// Replace <p><span>...</span></p>   with  <p>...</p>
+func (c *Cleaner) cleanParaSpans(doc *goquery.Document) *goquery.Document {
+	spans := doc.Find("span")
+	spans.Each(func(i int, s *goquery.Selection) {
+		parent := s.Parent()
+		if parent != nil && parent.Length() > 0 && parent.Get(0).DataAtom == atom.P {
+			replaceTagWithContents(s, whitelistedTextAtomTypes)
+		}
+	})
+	return doc
+}
+
+func (c *Cleaner) getFlushedBuffer(fragment string) []*html.Node {
+	var output []*html.Node
+	reader := strings.NewReader(fragment)
+	document, _ := html.Parse(reader)
+	body := document.FirstChild.LastChild
+	for c := body.FirstChild; c != nil; c = c.NextSibling {
+		output = append(output, c)
+		c.Parent = nil
+		c.PrevSibling = nil
+	}
+
+	for _, o := range output {
+		o.NextSibling = nil
+	}
+	return output
+}
+
+func (c *Cleaner) replaceWithPara(div *goquery.Selection) {
+	if div.Size() > 0 {
+		node := div.Get(0)
+		node.Data = atom.P.String()
+		node.DataAtom = atom.P
+		node.Attr = []html.Attribute{}
+	}
+}
+
+func (c *Cleaner) tabsAndNewLinesReplacements(text string) string {
+	text = strings.Replace(text, "\n", "\n\n", -1)
+	text = tabsRegEx.ReplaceAllString(text, "")
+	return text
+}
+
+func (c *Cleaner) convertDivsToParagraphs(doc *goquery.Document, domType string) *goquery.Document {
+	if c.config.debug {
+		log.Println("Starting to replace bad divs...")
+	}
+	badDivs := 0
+	convertedTextNodes := 0
+	divs := doc.Find(domType)
+
+	divs.Each(func(i int, div *goquery.Selection) {
+		divHTML, _ := div.Html()
+		if divToPElementsPattern.Match([]byte(divHTML)) {
+			c.replaceWithPara(div)
+			badDivs++
+		} else {
+			var replacementText []string
+			nodesToRemove := list.New()
+			children := div.Contents()
+			if c.config.debug {
+				log.Printf("Found %d children of div\n", children.Size())
+			}
+			children.EachWithBreak(func(i int, kid *goquery.Selection) bool {
+				text := kid.Text()
+				kidNode := kid.Get(0)
+				tag := kidNode.Data
+				if tag == text {
+					tag = "#text"
+				}
+				if tag == "#text" {
+					text = strings.Replace(text, "\n", "", -1)
+					text = tabsRegEx.ReplaceAllString(text, "")
+					if text == "" {
+						return true
+					}
+					if len(text) > 1 {
+						prev := kidNode.PrevSibling
+						if c.config.debug {
+							log.Printf("PARENT CLASS: %s NODENAME: %s\n", c.config.parser.name("class", div), tag)
+							log.Printf("TEXTREPLACE: %s\n", strings.Replace(text, "\n", "", -1))
+						}
+						if prev != nil && prev.DataAtom == atom.A {
+							nodeSelection := kid.HasNodes(prev)
+							html, _ := nodeSelection.Html()
+							replacementText = append(replacementText, html)
+							if c.config.debug {
+								log.Printf("SIBLING NODENAME ADDITION: %s TEXT: %s\n", prev.Data, html)
+							}
+						}
+						replacementText = append(replacementText, text)
+						nodesToRemove.PushBack(kidNode)
+						convertedTextNodes++
+					}
+
+				}
+				return true
+			})
+
+			/*
+			newNode := new(html.Node)
+			newNode.Type = html.ElementNode
+			newNode.Data = strings.Join(replacementText, "")
+			newNode.DataAtom = atom.P
+			*/
+/*
+			replacementText = strings.Replace(replacementText, "=C3=A8", "è")
+			replacementText = strings.Replace(replacementText, "=C3=A9", "é")
+*/
+			div.First().BeforeHtml("<p>" + strings.Join(replacementText, "") + "</p>")
+
+			for s := nodesToRemove.Front(); s != nil; s = s.Next() {
+				node := s.Value.(*html.Node)
+				if node != nil && node.Parent != nil {
+					node.Parent.RemoveChild(node)
+				}
+			}
+		}
+	})
+	if c.config.debug {
+		log.Printf("Found %d total divs with %d bad divs replaced and %d textnodes converted inside divs", divs.Size(), badDivs, convertedTextNodes)
+	}
+	return doc
+
+}
--- a/vendor/github.com/advancedlogic/GoOse/configuration.go
+++ b/vendor/github.com/advancedlogic/GoOse/configuration.go
@@ -0,0 +1,69 @@
+package goose
+
+import (
+	"time"
+)
+
+const defaultUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/534.52.7 (KHTML, like Gecko) Version/5.1.2 Safari/534.52.7"
+
+// Configuration is a wrapper for various config options
+type Configuration struct {
+	localStoragePath        string //not used in this version
+	imagesMinBytes          int    //not used in this version
+	targetLanguage          string
+	imageMagickConvertPath  string //not used in this version
+	imageMagickIdentifyPath string //not used in this version
+	browserUserAgent        string
+	debug                   bool
+	extractPublishDate      bool
+	additionalDataExtractor bool
+	enableImageFetching     bool
+	useMetaLanguage         bool
+
+	//path to the stopwords folder
+	stopWordsPath string
+	stopWords     StopWords
+	parser        *Parser
+
+	timeout time.Duration
+}
+
+// GetDefaultConfiguration returns safe default configuration options
+func GetDefaultConfiguration(args ...string) Configuration {
+	if len(args) == 0 {
+		return Configuration{
+			localStoragePath:        "",   //not used in this version
+			imagesMinBytes:          4500, //not used in this version
+			enableImageFetching:     true,
+			useMetaLanguage:         true,
+			targetLanguage:          "en",
+			imageMagickConvertPath:  "/usr/bin/convert",  //not used in this version
+			imageMagickIdentifyPath: "/usr/bin/identify", //not used in this version
+			browserUserAgent:        defaultUserAgent,
+			debug:                   false,
+			extractPublishDate:      true,
+			additionalDataExtractor: false,
+			stopWordsPath:           "resources/stopwords",
+			stopWords:               NewStopwords(), //TODO with path
+			parser:                  NewParser(),
+			timeout:                 time.Duration(5 * time.Second),
+		}
+	}
+	return Configuration{
+		localStoragePath:        "",   //not used in this version
+		imagesMinBytes:          4500, //not used in this version
+		enableImageFetching:     true,
+		useMetaLanguage:         true,
+		targetLanguage:          "en",
+		imageMagickConvertPath:  "/usr/bin/convert",  //not used in this version
+		imageMagickIdentifyPath: "/usr/bin/identify", //not used in this version
+		browserUserAgent:        defaultUserAgent,
+		debug:                   false,
+		extractPublishDate:      true,
+		additionalDataExtractor: false,
+		stopWordsPath:           "resources/stopwords",
+		stopWords:               NewStopwords(), //TODO with path
+		parser:                  NewParser(),
+		timeout:                 time.Duration(5 * time.Second),
+	}
+}
--- a/vendor/github.com/advancedlogic/GoOse/coverage.sh
+++ b/vendor/github.com/advancedlogic/GoOse/coverage.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# Run test coverage on each subdirectory and merge the coverage profile.
+echo "mode: count" > target/report/profile.cov
+
+# Standard go tooling behavior is to ignore dirs with leading underscors
+for dir in $(find . -maxdepth 10 -not -path './.git*' -not -path '*/_*' -type d); do
+    if ls $dir/*.go &> /dev/null; then
+        go test -covermode=count -coverprofile=$dir/profile.tmp $dir
+        if [ -f $dir/profile.tmp ]; then
+            cat $dir/profile.tmp | tail -n +2 >> target/report/profile.cov
+            rm $dir/profile.tmp
+        fi
+    fi
+done
+go tool cover -html target/report/profile.cov -o target/report/coverage.html
+
--- a/vendor/github.com/advancedlogic/GoOse/crawler.go
+++ b/vendor/github.com/advancedlogic/GoOse/crawler.go
@@ -0,0 +1,180 @@
+package goose
+
+import (
+	"errors"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+// Crawler can fetch the target HTML page
+type Crawler struct {
+	config  Configuration
+	Charset string
+}
+
+// NewCrawler returns a crawler object initialised with the URL and the [optional] raw HTML body
+func NewCrawler(config Configuration) Crawler {
+	return Crawler{
+		config:  config,
+		Charset: "",
+	}
+}
+
+func getCharsetFromContentType(cs string) string {
+	cs = strings.ToLower(strings.Replace(cs, " ", "", -1))
+	cs = strings.TrimPrefix(cs, "text/html;charset=")
+	cs = strings.TrimPrefix(cs, "text/xhtml;charset=")
+	cs = strings.TrimPrefix(cs, "application/xhtml+xml;charset=")
+	return NormaliseCharset(cs)
+}
+
+// SetCharset can be used to force a charset (e.g. when read from the HTTP headers)
+// rather than relying on the detection from the HTML meta tags
+func (c *Crawler) SetCharset(cs string) {
+	c.Charset = getCharsetFromContentType(cs)
+}
+
+// GetContentType returns the Content-Type string extracted from the meta tags
+func (c Crawler) GetContentType(document *goquery.Document) string {
+	var attr string
+	// <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+	document.Find("meta[http-equiv#=(?i)^Content\\-type$]").Each(func(i int, s *goquery.Selection) {
+		attr, _ = s.Attr("content")
+	})
+	return attr
+}
+
+// GetCharset returns a normalised charset string extracted from the meta tags
+func (c Crawler) GetCharset(document *goquery.Document) string {
+	// manually-provided charset (from HTTP headers?) takes priority
+	if "" != c.Charset {
+		return c.Charset
+	}
+
+	// <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+	ct := c.GetContentType(document)
+	if "" != ct && strings.Contains(strings.ToLower(ct), "charset") {
+		return getCharsetFromContentType(ct)
+	}
+
+	// <meta charset="utf-8">
+	selection := document.Find("meta").EachWithBreak(func(i int, s *goquery.Selection) bool {
+		_, exists := s.Attr("charset")
+		return !exists
+	})
+
+	if selection != nil {
+		cs, _ := selection.Attr("charset")
+		return NormaliseCharset(cs)
+	}
+
+	return ""
+}
+
+// Preprocess fetches the HTML page if needed, converts it to UTF-8 and applies
+// some text normalisation to guarantee better results when extracting the content
+func (c *Crawler) Preprocess(RawHTML string) (*goquery.Document, error) {
+	var err error
+
+	if RawHTML == "" {
+		return nil, errors.New("cannot process empty HTML content")
+	}
+
+	RawHTML = c.addSpacesBetweenTags(RawHTML)
+
+	reader := strings.NewReader(RawHTML)
+	document, err := goquery.NewDocumentFromReader(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	cs := c.GetCharset(document)
+	//log.Println("-------------------------------------------CHARSET:", cs)
+	if "" != cs && "UTF-8" != cs {
+		// the net/html parser and goquery require UTF-8 data
+		RawHTML = UTF8encode(RawHTML, cs)
+		reader = strings.NewReader(RawHTML)
+		if document, err = goquery.NewDocumentFromReader(reader); err != nil {
+			return nil, err
+		}
+	}
+
+	return document, nil
+}
+
+// Crawl fetches the HTML body and returns an Article
+func (c Crawler) Crawl(RawHTML string, url string) (*Article, error) {
+	article := new(Article)
+
+	document, err := c.Preprocess(RawHTML)
+	if nil != err {
+		return nil, err
+	}
+	if nil == document {
+		return article, nil
+	}
+	extractor := NewExtractor(c.config)
+	startTime := time.Now().UnixNano()
+
+	article.RawHTML, err = document.Html()
+	if nil != err {
+		return nil, err
+	}
+	article.FinalURL = url
+	article.Doc = document
+
+	article.TitleUnmodified = extractor.getTitleUnmodified(document)
+	article.Title = extractor.GetTitleFromUnmodifiedTitle(article.TitleUnmodified)
+	article.MetaLang = extractor.GetMetaLanguage(document)
+	article.MetaFavicon = extractor.GetFavicon(document)
+
+	article.MetaDescription = extractor.GetMetaContentWithSelector(document, "meta[name#=(?i)^description$]")
+	article.MetaKeywords = extractor.GetMetaContentWithSelector(document, "meta[name#=(?i)^keywords$]")
+	article.CanonicalLink = extractor.GetCanonicalLink(document)
+	if "" == article.CanonicalLink {
+		article.CanonicalLink = article.FinalURL
+	}
+	article.Domain = extractor.GetDomain(article.CanonicalLink)
+	article.Tags = extractor.GetTags(document)
+
+	if c.config.extractPublishDate {
+		if timestamp := extractor.GetPublishDate(document); timestamp != nil {
+			article.PublishDate = timestamp
+		}
+	}
+
+	cleaner := NewCleaner(c.config)
+	article.Doc = cleaner.Clean(article.Doc)
+
+	article.TopImage = OpenGraphResolver(document)
+	if article.TopImage == "" {
+		article.TopImage = WebPageResolver(article)
+	}
+
+	article.TopNode = extractor.CalculateBestNode(document)
+	if article.TopNode != nil {
+		article.TopNode = extractor.PostCleanup(article.TopNode)
+
+		article.CleanedText, article.Links = extractor.GetCleanTextAndLinks(article.TopNode, article.MetaLang)
+
+		videoExtractor := NewVideoExtractor()
+		article.Movies = videoExtractor.GetVideos(document)
+	}
+
+	article.Delta = time.Now().UnixNano() - startTime
+
+	return article, nil
+}
+
+// In many cases, like at the end of each <li> element or between </span><span> tags,
+// we need to add spaces, otherwise the text on either side will get joined together into one word.
+// This method also adds newlines after each </p> tag to preserve paragraphs.
+func (c Crawler) addSpacesBetweenTags(text string) string {
+	text = strings.Replace(text, "><", "> <", -1)
+	text = strings.Replace(text, "</blockquote>", "</blockquote>\n", -1)
+	text = strings.Replace(text, "<img ", "\n<img ", -1)
+	text = strings.Replace(text, "</li>", "</li>\n", -1)
+	return strings.Replace(text, "</p>", "</p>\n", -1)
+}
--- a/vendor/github.com/advancedlogic/GoOse/crawlershort.go
+++ b/vendor/github.com/advancedlogic/GoOse/crawlershort.go
@@ -0,0 +1,155 @@
+package goose
+
+import (
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/pkg/errors"
+)
+
+// Crawler can fetch the target HTML page
+type CrawlerShort struct {
+	config  Configuration
+	Charset string
+}
+
+// NewCrawler returns a crawler object initialised with the URL and the [optional] raw HTML body
+func NewCrawlerShort(config Configuration) CrawlerShort {
+	return CrawlerShort{
+		config:  config,
+		Charset: "",
+	}
+}
+
+// SetCharset can be used to force a charset (e.g. when read from the HTTP headers)
+// rather than relying on the detection from the HTML meta tags
+func (c *CrawlerShort) SetCharset(cs string) {
+	c.Charset = getCharsetFromContentType(cs)
+}
+
+// GetContentType returns the Content-Type string extracted from the meta tags
+func (c CrawlerShort) GetContentType(document *goquery.Document) string {
+	var attr string
+	// <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+	document.Find("meta[http-equiv#=(?i)^Content\\-type$]").Each(func(i int, s *goquery.Selection) {
+		attr, _ = s.Attr("content")
+	})
+	return attr
+}
+
+// GetCharset returns a normalised charset string extracted from the meta tags
+func (c CrawlerShort) GetCharset(document *goquery.Document) string {
+	// manually-provided charset (from HTTP headers?) takes priority
+	if "" != c.Charset {
+		return c.Charset
+	}
+
+	// <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+	ct := c.GetContentType(document)
+	if "" != ct && strings.Contains(strings.ToLower(ct), "charset") {
+		return getCharsetFromContentType(ct)
+	}
+
+	// <meta charset="utf-8">
+	selection := document.Find("meta").EachWithBreak(func(i int, s *goquery.Selection) bool {
+		_, exists := s.Attr("charset")
+		return !exists
+	})
+
+	if selection != nil {
+		cs, _ := selection.Attr("charset")
+		return NormaliseCharset(cs)
+	}
+
+	return ""
+}
+
+// Preprocess fetches the HTML page if needed, converts it to UTF-8 and applies
+// some text normalisation to guarantee better results when extracting the content
+func (c *CrawlerShort) Preprocess(RawHTML string) (*goquery.Document, error) {
+	var err error
+
+	RawHTML = c.addSpacesBetweenTags(RawHTML)
+
+	reader := strings.NewReader(RawHTML)
+	document, err := goquery.NewDocumentFromReader(reader)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not perform goquery.NewDocumentFromReader(reader)")
+	}
+
+	cs := c.GetCharset(document)
+	//log.Println("-------------------------------------------CHARSET:", cs)
+	if "" != cs && "UTF-8" != cs {
+		// the net/html parser and goquery require UTF-8 data
+		RawHTML = UTF8encode(RawHTML, cs)
+		reader = strings.NewReader(RawHTML)
+		if document, err = goquery.NewDocumentFromReader(reader); err != nil {
+			return nil, errors.Wrap(err, "could not perform goquery.NewDocumentFromReader(reader)")
+		}
+	}
+
+	return document, nil
+}
+
+// Crawl fetches the HTML body and returns an Article
+func (c CrawlerShort) Crawl(RawHTML, url string) (*Article, error) {
+	article := new(Article)
+
+	document, err := c.Preprocess(RawHTML)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not Preprocess RawHTML")
+	}
+	if document == nil {
+		return article, nil
+	}
+
+	extractor := NewExtractor(c.config)
+
+	startTime := time.Now().UnixNano()
+
+	article.RawHTML, err = document.Html()
+	if err != nil {
+		return nil, errors.Wrap(err, "could not get html from document")
+	}
+	article.FinalURL = url
+
+	article.Title = extractor.GetTitle(document)
+	article.MetaDescription = extractor.GetMetaContentWithSelector(document, "meta[name#=(?i)^description$]")
+
+	if c.config.extractPublishDate {
+		if timestamp := extractor.GetPublishDate(document); timestamp != nil {
+			article.PublishDate = timestamp
+		}
+	}
+
+	cleaner := NewCleaner(c.config)
+	article.Doc = cleaner.Clean(article.Doc)
+
+	article.TopImage = OpenGraphResolver(document)
+	if article.TopImage == "" {
+		article.TopImage = WebPageResolver(article)
+	}
+
+	article.TopNode = extractor.CalculateBestNode(document)
+	if article.TopNode != nil {
+		article.TopNode = extractor.PostCleanup(article.TopNode)
+
+		article.CleanedText, article.Links = extractor.GetCleanTextAndLinks(article.TopNode, article.MetaLang)
+
+	}
+	article.Delta = time.Now().UnixNano() - startTime
+
+	return article, nil
+}
+
+// In many cases, like at the end of each <li> element or between </span><span> tags,
+// we need to add spaces, otherwise the text on either side will get joined together into one word.
+// This method also adds newlines after each </p> tag to preserve paragraphs.
+func (c CrawlerShort) addSpacesBetweenTags(text string) string {
+	text = strings.Replace(text, "><", "> <", -1)
+	text = strings.Replace(text, "</blockquote>", "</blockquote>\n", -1)
+	text = strings.Replace(text, "<img ", "\n<img ", -1)
+	text = strings.Replace(text, "</li>", "</li>\n", -1)
+	return strings.Replace(text, "</p>", "</p>\n", -1)
+}
--- a/vendor/github.com/advancedlogic/GoOse/doc.go
+++ b/vendor/github.com/advancedlogic/GoOse/doc.go
@@ -0,0 +1,23 @@
+/*
+This is a golang port of "Goose" originally licensed to Gravity.com
+under one or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.
+
+Golang port was written by Antonio Linari
+
+Gravity.com licenses this file
+to you under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package goose
--- a/vendor/github.com/advancedlogic/GoOse/extractor.go
+++ b/vendor/github.com/advancedlogic/GoOse/extractor.go
@@ -0,0 +1,754 @@
+package goose
+
+import (
+	"container/list"
+	"log"
+	"math"
+	"net/url"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/araddon/dateparse"
+	"github.com/fatih/set"
+	"github.com/gigawattio/window"
+	"github.com/jaytaylor/html2text"
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/atom"
+)
+
+const defaultLanguage = "en"
+
+var motleyReplacement = "&#65533;" // U+FFFD (decimal 65533) is the "replacement character".
+//var escapedFragmentReplacement = regexp.MustCompile("#!")
+//var titleReplacements = regexp.MustCompile("&raquo;")
+
+var titleDelimiters = []string{
+	"|",
+	" - ",
+	" — ",
+	"»",
+	":",
+}
+
+var aRelTagSelector = "a[rel=tag]"
+var aHrefTagSelector = [...]string{"/tag/", "/tags/", "/topic/", "?keyword"}
+
+//var langRegEx = "^[A-Za-z]{2}$"
+
+// ContentExtractor can parse the HTML and fetch various properties
+type ContentExtractor struct {
+	config Configuration
+}
+
+// NewExtractor returns a configured HTML parser
+func NewExtractor(config Configuration) ContentExtractor {
+	return ContentExtractor{
+		config: config,
+	}
+}
+
+//if the article has a title set in the source, use that
+func (extr *ContentExtractor) getTitleUnmodified(document *goquery.Document) string {
+	title := ""
+
+	titleElement := document.Find("title")
+	if titleElement != nil && titleElement.Size() > 0 {
+		title = titleElement.Text()
+	}
+
+	if title == "" {
+		ogTitleElement := document.Find(`meta[property="og:title"]`)
+		if ogTitleElement != nil && ogTitleElement.Size() > 0 {
+			title, _ = ogTitleElement.Attr("content")
+		}
+	}
+
+	if title == "" {
+		titleElement = document.Find("post-title,headline")
+		if titleElement == nil || titleElement.Size() == 0 {
+			return title
+		}
+		title = titleElement.Text()
+	}
+	return title
+}
+
+// GetTitleFromUnmodifiedTitle returns the title from the unmodified one
+func (extr *ContentExtractor) GetTitleFromUnmodifiedTitle(title string) string {
+	for _, delimiter := range titleDelimiters {
+		if strings.Contains(title, delimiter) {
+			title = extr.splitTitle(strings.Split(title, delimiter))
+			break
+		}
+	}
+
+	title = strings.Replace(title, motleyReplacement, "", -1)
+
+	if extr.config.debug {
+		log.Printf("Page title is %s\n", title)
+	}
+
+	return strings.TrimSpace(title)
+}
+
+// GetTitle returns the title set in the source, if the article has one
+func (extr *ContentExtractor) GetTitle(document *goquery.Document) string {
+	title := extr.getTitleUnmodified(document)
+	return extr.GetTitleFromUnmodifiedTitle(title)
+}
+
+func (extr *ContentExtractor) splitTitle(titles []string) string {
+	largeTextLength := 0
+	largeTextIndex := 0
+	for i, current := range titles {
+		if len(current) > largeTextLength {
+			largeTextLength = len(current)
+			largeTextIndex = i
+		}
+	}
+	title := titles[largeTextIndex]
+	title = strings.Replace(title, "&raquo;", "»", -1)
+	return title
+}
+
+// GetMetaLanguage returns the meta language set in the source, if the article has one
+func (extr *ContentExtractor) GetMetaLanguage(document *goquery.Document) string {
+	var language string
+	shtml := document.Find("html")
+	attr, _ := shtml.Attr("lang")
+	if attr == "" {
+		attr, _ = document.Attr("lang")
+	}
+	if attr == "" {
+		selection := document.Find("meta").EachWithBreak(func(i int, s *goquery.Selection) bool {
+			var exists bool
+			attr, exists = s.Attr("http-equiv")
+			if exists && attr == "content-language" {
+				return false
+			}
+			return true
+		})
+		if selection != nil {
+			attr, _ = selection.Attr("content")
+		}
+	}
+	idx := strings.LastIndex(attr, "-")
+	if idx == -1 {
+		language = attr
+	} else {
+		language = attr[0:idx]
+	}
+
+	_, ok := sw[language]
+
+	if language == "" || !ok {
+		language = extr.config.stopWords.SimpleLanguageDetector(shtml.Text())
+		if language == "" {
+			language = defaultLanguage
+		}
+	}
+
+	extr.config.targetLanguage = language
+	return language
+}
+
+// GetFavicon returns the favicon set in the source, if the article has one
+func (extr *ContentExtractor) GetFavicon(document *goquery.Document) string {
+	favicon := ""
+	document.Find("link").EachWithBreak(func(i int, s *goquery.Selection) bool {
+		attr, exists := s.Attr("rel")
+		if exists && strings.Contains(attr, "icon") {
+			favicon, _ = s.Attr("href")
+			return false
+		}
+		return true
+	})
+	return favicon
+}
+
+// GetMetaContentWithSelector returns the content attribute of meta tag matching the selector
+func (extr *ContentExtractor) GetMetaContentWithSelector(document *goquery.Document, selector string) string {
+	selection := document.Find(selector)
+	content, _ := selection.Attr("content")
+	return strings.TrimSpace(content)
+}
+
+// GetMetaContent returns the content attribute of meta tag with the given property name
+func (extr *ContentExtractor) GetMetaContent(document *goquery.Document, metaName string) string {
+	content := ""
+	document.Find("meta").EachWithBreak(func(i int, s *goquery.Selection) bool {
+		attr, exists := s.Attr("name")
+		if exists && attr == metaName {
+			content, _ = s.Attr("content")
+			return false
+		}
+		attr, exists = s.Attr("itemprop")
+		if exists && attr == metaName {
+			content, _ = s.Attr("content")
+			return false
+		}
+		return true
+	})
+	return content
+}
+
+// GetMetaContents returns all the meta tags as name->content pairs
+func (extr *ContentExtractor) GetMetaContents(document *goquery.Document, metaNames *set.Set) map[string]string {
+	contents := make(map[string]string)
+	counter := metaNames.Size()
+	document.Find("meta").EachWithBreak(func(i int, s *goquery.Selection) bool {
+		attr, exists := s.Attr("name")
+		if exists && metaNames.Has(attr) {
+			content, _ := s.Attr("content")
+			contents[attr] = content
+			counter--
+			if counter < 0 {
+				return false
+			}
+		}
+		return true
+	})
+	return contents
+}
+
+// GetMetaDescription returns the meta description set in the source, if the article has one
+func (extr *ContentExtractor) GetMetaDescription(document *goquery.Document) string {
+	return extr.GetMetaContent(document, "description")
+}
+
+// GetMetaKeywords returns the meta keywords set in the source, if the article has them
+func (extr *ContentExtractor) GetMetaKeywords(document *goquery.Document) string {
+	return extr.GetMetaContent(document, "keywords")
+}
+
+// GetMetaAuthor returns the meta author set in the source, if the article has one
+func (extr *ContentExtractor) GetMetaAuthor(document *goquery.Document) string {
+	return extr.GetMetaContent(document, "author")
+}
+
+// GetMetaContentLocation returns the meta content location set in the source, if the article has one
+func (extr *ContentExtractor) GetMetaContentLocation(document *goquery.Document) string {
+	return extr.GetMetaContent(document, "contentLocation")
+}
+
+// GetCanonicalLink returns the meta canonical link set in the source
+func (extr *ContentExtractor) GetCanonicalLink(document *goquery.Document) string {
+	metas := document.Find("link[rel=canonical]")
+	if metas.Length() > 0 {
+		meta := metas.First()
+		href, _ := meta.Attr("href")
+		href = strings.Trim(href, "\n")
+		href = strings.Trim(href, " ")
+		if href != "" {
+			return href
+		}
+	}
+	return ""
+}
+
+// GetDomain extracts the domain from a link
+func (extr *ContentExtractor) GetDomain(canonicalLink string) string {
+	u, err := url.Parse(canonicalLink)
+	if err == nil {
+		return u.Host
+	}
+	return ""
+}
+
+// GetTags returns the tags set in the source, if the article has them
+func (extr *ContentExtractor) GetTags(document *goquery.Document) *set.Set {
+	tags := set.New(set.ThreadSafe).(*set.Set)
+	selections := document.Find(aRelTagSelector)
+	selections.Each(func(i int, s *goquery.Selection) {
+		tags.Add(s.Text())
+	})
+	selections = document.Find("a")
+	selections.Each(func(i int, s *goquery.Selection) {
+		href, exists := s.Attr("href")
+		if exists {
+			for _, part := range aHrefTagSelector {
+				if strings.Contains(href, part) {
+					tags.Add(s.Text())
+				}
+			}
+		}
+	})
+
+	return tags
+}
+
+// GetPublishDate returns the publication date, if one can be located.
+func (extr *ContentExtractor) GetPublishDate(document *goquery.Document) *time.Time {
+	raw, err := document.Html()
+	if err != nil {
+		log.Printf("Error converting document HTML nodes to raw HTML: %s (publish date detection aborted)\n", err)
+		return nil
+	}
+
+	text, err := html2text.FromString(raw)
+	if err != nil {
+		log.Printf("Error converting document HTML to plaintext: %s (publish date detection aborted)\n", err)
+		return nil
+	}
+
+	text = strings.ToLower(text)
+
+	// Simplify months because the dateparse pkg only handles abbreviated.
+	for k, v := range map[string]string{
+		"january":  "jan",
+		"march":    "mar",
+		"february": "feb",
+		"april":    "apr",
+		// "may":       "may", // Pointless.
+		"june":      "jun",
+		"august":    "aug",
+		"september": "sep",
+		"sept":      "sep",
+		"october":   "oct",
+		"november":  "nov",
+		"december":  "dec",
+		"th,":       ",", // Strip day number suffixes.
+		"rd,":       ",",
+	} {
+		text = strings.Replace(text, k, v, -1)
+	}
+	text = strings.Replace(text, "\n", " ", -1)
+	text = regexp.MustCompile(" +").ReplaceAllString(text, " ")
+
+	tuple1 := strings.Split(text, " ")
+
+	var (
+		expr  = regexp.MustCompile("[0-9]")
+		ts    time.Time
+		found bool
+	)
+	for _, n := range []int{3, 4, 5, 2, 6} {
+		for _, win := range window.Rolling(tuple1, n) {
+			if !expr.MatchString(strings.Join(win, " ")) {
+				continue
+			}
+
+			input := strings.Join(win, " ")
+			ts, err = dateparse.ParseAny(input)
+			if err == nil && ts.Year() > 0 && ts.Month() > 0 && ts.Day() > 0 {
+				found = true
+				break
+			}
+
+			// Try injecting a comma for dateparse.
+			win[1] = win[1] + ","
+			input = strings.Join(win, " ")
+			ts, err = dateparse.ParseAny(input)
+			if err == nil && ts.Year() > 0 && ts.Month() > 0 && ts.Day() > 0 {
+				found = true
+				break
+			}
+		}
+		if found {
+			break
+		}
+	}
+
+	if found {
+		return &ts
+	}
+	return nil
+}
+
+// GetCleanTextAndLinks parses the main HTML node for text and links
+func (extr *ContentExtractor) GetCleanTextAndLinks(topNode *goquery.Selection, lang string) (string, []string) {
+	outputFormatter := new(outputFormatter)
+	outputFormatter.config = extr.config
+	return outputFormatter.getFormattedText(topNode, lang)
+}
+
+// CalculateBestNode checks for the HTML node most likely to contain the main content.
+//we're going to start looking for where the clusters of paragraphs are. We'll score a cluster based on the number of stopwords
+//and the number of consecutive paragraphs together, which should form the cluster of text that this node is around
+//also store on how high up the paragraphs are, comments are usually at the bottom and should get a lower score
+func (extr *ContentExtractor) CalculateBestNode(document *goquery.Document) *goquery.Selection {
+	var topNode *goquery.Selection
+	nodesToCheck := extr.nodesToCheck(document)
+	if extr.config.debug {
+		log.Printf("Nodes to check %d\n", len(nodesToCheck))
+	}
+	startingBoost := 1.0
+	cnt := 0
+	i := 0
+	parentNodes := set.New(set.ThreadSafe).(*set.Set)
+	nodesWithText := list.New()
+	for _, node := range nodesToCheck {
+		textNode := node.Text()
+		ws := extr.config.stopWords.stopWordsCount(extr.config.targetLanguage, textNode)
+		highLinkDensity := extr.isHighLinkDensity(node)
+		if ws.stopWordCount > 2 && !highLinkDensity {
+			nodesWithText.PushBack(node)
+		}
+	}
+	nodesNumber := nodesWithText.Len()
+	negativeScoring := 0
+	bottomNegativeScoring := float64(nodesNumber) * 0.25
+
+	if extr.config.debug {
+		log.Printf("About to inspect num of nodes with text %d\n", nodesNumber)
+	}
+
+	for n := nodesWithText.Front(); n != nil; n = n.Next() {
+		node := n.Value.(*goquery.Selection)
+		boostScore := 0.0
+		if extr.isBoostable(node) {
+			if cnt >= 0 {
+				boostScore = float64((1.0 / startingBoost) * 50)
+				startingBoost++
+			}
+		}
+
+		if nodesNumber > 15 {
+			if float64(nodesNumber-i) <= bottomNegativeScoring {
+				booster := bottomNegativeScoring - float64(nodesNumber-i)
+				boostScore = -math.Pow(booster, 2.0)
+				negScore := math.Abs(boostScore) + float64(negativeScoring)
+				if negScore > 40 {
+					boostScore = 5.0
+				}
+			}
+		}
+
+		if extr.config.debug {
+			log.Printf("Location Boost Score %1.5f on iteration %d id='%s' class='%s'\n", boostScore, i, extr.config.parser.name("id", node), extr.config.parser.name("class", node))
+		}
+		textNode := node.Text()
+		ws := extr.config.stopWords.stopWordsCount(extr.config.targetLanguage, textNode)
+		upScore := ws.stopWordCount + int(boostScore)
+		parentNode := node.Parent()
+		extr.updateScore(parentNode, upScore)
+		extr.updateNodeCount(parentNode, 1)
+		if !parentNodes.Has(parentNode) {
+			parentNodes.Add(parentNode)
+		}
+		parentParentNode := parentNode.Parent()
+		if parentParentNode != nil {
+			extr.updateNodeCount(parentParentNode, 1)
+			extr.updateScore(parentParentNode, upScore/2.0)
+			if !parentNodes.Has(parentParentNode) {
+				parentNodes.Add(parentParentNode)
+			}
+		}
+		cnt++
+		i++
+	}
+
+	topNodeScore := 0
+	parentNodesArray := parentNodes.List()
+	for _, p := range parentNodesArray {
+		e := p.(*goquery.Selection)
+		if extr.config.debug {
+			log.Printf("ParentNode: score=%s nodeCount=%s id='%s' class='%s'\n", extr.config.parser.name("gravityScore", e), extr.config.parser.name("gravityNodes", e), extr.config.parser.name("id", e), extr.config.parser.name("class", e))
+		}
+		score := extr.getScore(e)
+		if score >= topNodeScore {
+			topNode = e
+			topNodeScore = score
+		}
+		if topNode == nil {
+			topNode = e
+		}
+	}
+	return topNode
+}
+
+//returns the gravityScore as an integer from this node
+func (extr *ContentExtractor) getScore(node *goquery.Selection) int {
+	return extr.getNodeGravityScore(node)
+}
+
+func (extr *ContentExtractor) getNodeGravityScore(node *goquery.Selection) int {
+	grvScoreString, exists := node.Attr("gravityScore")
+	if !exists {
+		return 0
+	}
+	grvScore, err := strconv.Atoi(grvScoreString)
+	if err != nil {
+		return 0
+	}
+	return grvScore
+}
+
+//adds a score to the gravityScore Attribute we put on divs
+//we'll get the current score then add the score we're passing in to the current
+func (extr *ContentExtractor) updateScore(node *goquery.Selection, addToScore int) {
+	currentScore := 0
+	var err error
+	scoreString, _ := node.Attr("gravityScore")
+	if scoreString != "" {
+		currentScore, err = strconv.Atoi(scoreString)
+		if err != nil {
+			currentScore = 0
+		}
+	}
+	newScore := currentScore + addToScore
+	extr.config.parser.setAttr(node, "gravityScore", strconv.Itoa(newScore))
+}
+
+//stores how many decent nodes are under a parent node
+func (extr *ContentExtractor) updateNodeCount(node *goquery.Selection, addToCount int) {
+	currentScore := 0
+	var err error
+	scoreString, _ := node.Attr("gravityNodes")
+	if scoreString != "" {
+		currentScore, err = strconv.Atoi(scoreString)
+		if err != nil {
+			currentScore = 0
+		}
+	}
+	newScore := currentScore + addToCount
+	extr.config.parser.setAttr(node, "gravityNodes", strconv.Itoa(newScore))
+}
+
+//a lot of times the first paragraph might be the caption under an image so we'll want to make sure if we're going to
+//boost a parent node that it should be connected to other paragraphs, at least for the first n paragraphs
+//so we'll want to make sure that the next sibling is a paragraph and has at least some substantial weight to it
+func (extr *ContentExtractor) isBoostable(node *goquery.Selection) bool {
+	stepsAway := 0
+	next := node.Next()
+	for next != nil && stepsAway < node.Siblings().Length() {
+		currentNodeTag := node.Get(0).DataAtom.String()
+		if currentNodeTag == "p" {
+			if stepsAway >= 3 {
+				if extr.config.debug {
+					log.Println("Next paragraph is too far away, not boosting")
+				}
+				return false
+			}
+
+			paraText := node.Text()
+			ws := extr.config.stopWords.stopWordsCount(extr.config.targetLanguage, paraText)
+			if ws.stopWordCount > 5 {
+				if extr.config.debug {
+					log.Println("We're gonna boost this node, seems content")
+				}
+				return true
+			}
+		}
+
+		stepsAway++
+		next = next.Next()
+	}
+
+	return false
+}
+
+//returns a list of nodes we want to search on like paragraphs and tables
+func (extr *ContentExtractor) nodesToCheck(doc *goquery.Document) []*goquery.Selection {
+	var output []*goquery.Selection
+	tags := []string{"p", "pre", "td"}
+	for _, tag := range tags {
+		selections := doc.Children().Find(tag)
+		if selections != nil {
+			selections.Each(func(i int, s *goquery.Selection) {
+				output = append(output, s)
+			})
+		}
+	}
+	return output
+}
+
+//checks the density of links within a node, is there not much text and most of it contains bad links?
+//if so it's no good
+func (extr *ContentExtractor) isHighLinkDensity(node *goquery.Selection) bool {
+	links := node.Find("a")
+	if links == nil || links.Size() == 0 {
+		return false
+	}
+	text := node.Text()
+	words := strings.Split(text, " ")
+	nwords := len(words)
+	var sb []string
+	links.Each(func(i int, s *goquery.Selection) {
+		linkText := s.Text()
+		sb = append(sb, linkText)
+	})
+	linkText := strings.Join(sb, "")
+	linkWords := strings.Split(linkText, " ")
+	nlinkWords := len(linkWords)
+	nlinks := links.Size()
+	linkDivisor := float64(nlinkWords) / float64(nwords)
+	score := linkDivisor * float64(nlinks)
+
+	if extr.config.debug {
+		var logText string
+		if len(node.Text()) >= 51 {
+			logText = node.Text()[0:50]
+		} else {
+			logText = node.Text()
+		}
+		log.Printf("Calculated link density score as %1.5f for node %s\n", score, logText)
+	}
+	if score > 1.0 {
+		return true
+	}
+	return false
+}
+
+func (extr *ContentExtractor) isTableAndNoParaExist(selection *goquery.Selection) bool {
+	subParagraph := selection.Find("p")
+	subParagraph.Each(func(i int, s *goquery.Selection) {
+		txt := s.Text()
+		if len(txt) < 25 {
+			node := s.Get(0)
+			parent := node.Parent
+			parent.RemoveChild(node)
+		}
+	})
+
+	subParagraph2 := selection.Find("p")
+	if subParagraph2.Length() == 0 && selection.Get(0).DataAtom.String() != "td" {
+		return true
+	}
+	return false
+}
+
+func (extr *ContentExtractor) isNodescoreThresholdMet(node *goquery.Selection, e *goquery.Selection) bool {
+	topNodeScore := extr.getNodeGravityScore(node)
+	currentNodeScore := extr.getNodeGravityScore(e)
+	threasholdScore := float64(topNodeScore) * 0.08
+	if (float64(currentNodeScore) < threasholdScore) && e.Get(0).DataAtom.String() != "td" {
+		return false
+	}
+	return true
+}
+
+//we could have long articles that have tons of paragraphs so if we tried to calculate the base score against
+//the total text score of those paragraphs it would be unfair. So we need to normalize the score based on the average scoring
+//of the paragraphs within the top node. For example if our total score of 10 paragraphs was 1000 but each had an average value of
+//100 then 100 should be our base.
+func (extr *ContentExtractor) getSiblingsScore(topNode *goquery.Selection) int {
+	base := 100000
+	paragraphNumber := 0
+	paragraphScore := 0
+	nodesToCheck := topNode.Find("p")
+	nodesToCheck.Each(func(i int, s *goquery.Selection) {
+		textNode := s.Text()
+		ws := extr.config.stopWords.stopWordsCount(extr.config.targetLanguage, textNode)
+		highLinkDensity := extr.isHighLinkDensity(s)
+		if ws.stopWordCount > 2 && !highLinkDensity {
+			paragraphNumber++
+			paragraphScore += ws.stopWordCount
+		}
+	})
+	if paragraphNumber > 0 {
+		base = paragraphScore / paragraphNumber
+	}
+	return base
+}
+
+func (extr *ContentExtractor) getSiblingsContent(currentSibling *goquery.Selection, baselinescoreSiblingsPara float64) []*goquery.Selection {
+	var ps []*goquery.Selection
+	if currentSibling.Get(0).DataAtom.String() == "p" && len(currentSibling.Text()) > 0 {
+		ps = append(ps, currentSibling)
+		return ps
+	}
+
+	potentialParagraphs := currentSibling.Find("p")
+	potentialParagraphs.Each(func(i int, s *goquery.Selection) {
+		text := s.Text()
+		if len(text) > 0 {
+			ws := extr.config.stopWords.stopWordsCount(extr.config.targetLanguage, text)
+			paragraphScore := ws.stopWordCount
+			siblingBaselineScore := 0.30
+			highLinkDensity := extr.isHighLinkDensity(s)
+			score := siblingBaselineScore * baselinescoreSiblingsPara
+			if score < float64(paragraphScore) && !highLinkDensity {
+				node := new(html.Node)
+				node.Type = html.TextNode
+				node.Data = text
+				node.DataAtom = atom.P
+				nodes := make([]*html.Node, 1)
+				nodes[0] = node
+				newSelection := new(goquery.Selection)
+				newSelection.Nodes = nodes
+				ps = append(ps, newSelection)
+			}
+		}
+
+	})
+	return ps
+}
+
+func (extr *ContentExtractor) walkSiblings(node *goquery.Selection) []*goquery.Selection {
+	currentSibling := node.Prev()
+	var b []*goquery.Selection
+	for currentSibling.Length() != 0 {
+		b = append(b, currentSibling)
+		previousSibling := currentSibling.Prev()
+		currentSibling = previousSibling
+	}
+	return b
+}
+
+//adds any siblings that may have a decent score to this node
+func (extr *ContentExtractor) addSiblings(topNode *goquery.Selection) *goquery.Selection {
+	if extr.config.debug {
+		log.Println("Starting to add siblings")
+	}
+	baselinescoreSiblingsPara := extr.getSiblingsScore(topNode)
+	results := extr.walkSiblings(topNode)
+	for _, currentNode := range results {
+		ps := extr.getSiblingsContent(currentNode, float64(baselinescoreSiblingsPara))
+		for _, p := range ps {
+			nodes := make([]*html.Node, len(topNode.Nodes)+1)
+			nodes[0] = p.Get(0)
+			for i, node := range topNode.Nodes {
+				nodes[i+1] = node
+			}
+			topNode.Nodes = nodes
+		}
+	}
+	return topNode
+}
+
+//PostCleanup removes any divs that looks like non-content, clusters of links, or paras with no gusto
+func (extr *ContentExtractor) PostCleanup(targetNode *goquery.Selection) *goquery.Selection {
+	if extr.config.debug {
+		log.Println("Starting cleanup Node")
+	}
+	node := extr.addSiblings(targetNode)
+	children := node.Children()
+	children.Each(func(i int, s *goquery.Selection) {
+		tag := s.Get(0).DataAtom.String()
+		if tag != "p" {
+			if extr.config.debug {
+				log.Printf("CLEANUP  NODE: %s class: %s\n", extr.config.parser.name("id", s), extr.config.parser.name("class", s))
+			}
+			//if extr.isHighLinkDensity(s) || extr.isTableAndNoParaExist(s) || !extr.isNodescoreThresholdMet(node, s) {
+			if extr.isHighLinkDensity(s) {
+				extr.config.parser.removeNode(s)
+				return
+			}
+
+			subParagraph := s.Find("p")
+			subParagraph.Each(func(j int, e *goquery.Selection) {
+				if len(e.Text()) < 25 {
+					extr.config.parser.removeNode(e)
+				}
+			})
+
+			subParagraph2 := s.Find("p")
+			if subParagraph2.Length() == 0 && tag != "td" {
+				if extr.config.debug {
+					log.Println("Removing node because it doesn't have any paragraphs")
+				}
+				extr.config.parser.removeNode(s)
+			} else {
+				if extr.config.debug {
+					log.Println("Not removing TD node")
+				}
+			}
+			return
+		}
+	})
+	return node
+}
--- a/vendor/github.com/advancedlogic/GoOse/go.mod
+++ b/vendor/github.com/advancedlogic/GoOse/go.mod
@@ -0,0 +1,22 @@
+module github.com/advancedlogic/GoOse
+
+go 1.13
+
+require (
+	github.com/PuerkitoBio/goquery v1.4.1
+	github.com/andybalholm/cascadia v1.0.0 // indirect
+	github.com/araddon/dateparse v0.0.0-20180729174819-cfd92a431d0e
+	github.com/fatih/set v0.2.1
+	github.com/gigawattio/window v0.0.0-20180317192513-0f5467e35573
+	github.com/go-resty/resty/v2 v2.0.0
+	github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0
+	github.com/mattn/go-runewidth v0.0.3 // indirect
+	github.com/olekukonko/tablewriter v0.0.0-20180506121414-d4647c9c7a84 // indirect
+	github.com/pkg/errors v0.8.1
+	github.com/simplereach/timeutils v1.2.0 // indirect
+	github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
+	github.com/stretchr/testify v1.4.0 // indirect
+	golang.org/x/net v0.0.0-20190628185345-da137c7871d7
+	golang.org/x/text v0.3.0
+	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
+)
--- a/vendor/github.com/advancedlogic/GoOse/go.sum
+++ b/vendor/github.com/advancedlogic/GoOse/go.sum
@@ -0,0 +1,44 @@
+github.com/PuerkitoBio/goquery v1.4.1 h1:smcIRGdYm/w7JSbcdeLHEMzxmsBQvl8lhf0dSw2nzMI=
+github.com/PuerkitoBio/goquery v1.4.1/go.mod h1:T9ezsOHcCrDCgA8aF1Cqr3sSYbO/xgdy8/R/XiIMAhA=
+github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
+github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/araddon/dateparse v0.0.0-20180729174819-cfd92a431d0e h1:s05JG2GwtJMHaPcXDpo4V35TFgyYZzNsmBlSkHPEbeg=
+github.com/araddon/dateparse v0.0.0-20180729174819-cfd92a431d0e/go.mod h1:SLqhdZcd+dF3TEVL2RMoob5bBP5R1P1qkox+HtCBgGI=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fatih/set v0.2.1 h1:nn2CaJyknWE/6txyUDGwysr3G5QC6xWB/PtVjPBbeaA=
+github.com/fatih/set v0.2.1/go.mod h1:+RKtMCH+favT2+3YecHGxcc0b4KyVWA1QWWJUs4E0CI=
+github.com/gigawattio/window v0.0.0-20180317192513-0f5467e35573 h1:u8AQ9bPa9oC+8/A/jlWouakhIvkFfuxgIIRjiy8av7I=
+github.com/gigawattio/window v0.0.0-20180317192513-0f5467e35573/go.mod h1:eBvb3i++NHDH4Ugo9qCvMw8t0mTSctaEa5blJbWcNxs=
+github.com/go-resty/resty/v2 v2.0.0 h1:9Nq/U+V4xsoDnDa/iTrABDWUCuk3Ne92XFHPe6dKWUc=
+github.com/go-resty/resty/v2 v2.0.0/go.mod h1:dZGr0i9PLlaaTD4H/hoZIDjQ+r6xq8mgbRzHZf7f2J8=
+github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0 h1:xqgexXAGQgY3HAjNPSaCqn5Aahbo5TKsmhp8VRfr1iQ=
+github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk=
+github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4=
+github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/olekukonko/tablewriter v0.0.0-20180506121414-d4647c9c7a84 h1:fiKJgB4JDUd43CApkmCeTSQlWjtTtABrU2qsgbuP0BI=
+github.com/olekukonko/tablewriter v0.0.0-20180506121414-d4647c9c7a84/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/simplereach/timeutils v1.2.0 h1:btgOAlu9RW6de2r2qQiONhjgxdAG7BL6je0G6J/yPnA=
+github.com/simplereach/timeutils v1.2.0/go.mod h1:VVbQDfN/FHRZa1LSqcwo4kNZ62OOyqLLGQKYB3pB0Q8=
+github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo=
+github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
--- a/vendor/github.com/advancedlogic/GoOse/goose.go
+++ b/vendor/github.com/advancedlogic/GoOse/goose.go
@@ -0,0 +1,34 @@
+package goose
+
+import (
+	"github.com/pkg/errors"
+)
+
+// Goose is the main entry point of the program
+type Goose struct {
+	config Configuration
+}
+
+// New returns a new instance of the article extractor
+func New(args ...string) Goose {
+	return Goose{
+		config: GetDefaultConfiguration(args...),
+	}
+}
+
+// ExtractFromURL follows the URL, fetches the HTML page and returns an article object
+func (g Goose) ExtractFromURL(url string) (*Article, error) {
+	HtmlRequester := NewHtmlRequester(g.config)
+	html, err := HtmlRequester.fetchHTML(url)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not get htnk from site")
+	}
+	cc := NewCrawler(g.config)
+	return cc.Crawl(html, url)
+}
+
+// ExtractFromRawHTML returns an article object from the raw HTML content
+func (g Goose) ExtractFromRawHTML(RawHTML string, url string) (*Article, error) {
+	cc := NewCrawler(g.config)
+	return cc.Crawl(RawHTML, url)
+}
--- a/vendor/github.com/advancedlogic/GoOse/goose.json
+++ b/vendor/github.com/advancedlogic/GoOse/goose.json
@@ -0,0 +1,37 @@
+/*
+This is a golang port of "Goose" originaly licensed to Gravity.com
+under one or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.
+
+Golang port was written by Antonio Linari
+
+Gravity.com licenses this file
+to you under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+//JSON Configuration file
+{
+  "localStoragePath": "",
+  "imagesMinBytes": 4500,
+  "enableImageFetching": true,
+  "useMetaLanguage": true,
+  "targetLanguage": "en",
+  "imageMagickConvertPath": "/usr/bin/convert",
+  "imageMagickIdentifyPath": "/usr/bin/identify",
+  "browserUserAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/534.52.7 (KHTML, like Gecko) Version/5.1.2 Safari/534.52.7",
+  "debug": false,
+  "extractPublishDate": false,
+  "additionalDataExtractor": false,
+  "stopWordsPath": "resources/stopwords"
+}
--- a/vendor/github.com/advancedlogic/GoOse/html.go
+++ b/vendor/github.com/advancedlogic/GoOse/html.go
@@ -0,0 +1,47 @@
+package goose
+
+import (
+	resty "github.com/go-resty/resty/v2"
+	"github.com/pkg/errors"
+)
+
+type HtmlRequester interface {
+	fetchHTML(string) (string, error)
+}
+
+// Crawler can fetch the target HTML page
+type htmlrequester struct {
+	config Configuration
+}
+
+// NewCrawler returns a crawler object initialised with the URL and the [optional] raw HTML body
+func NewHtmlRequester(config Configuration) HtmlRequester {
+	return htmlrequester{
+		config: config,
+	}
+}
+
+func (hr htmlrequester) fetchHTML(url string) (string, error) {
+	client := resty.New()
+	client.SetTimeout(hr.config.timeout)
+	resp, err := client.R().
+		SetHeader("Content-Type", "text/html").
+		SetHeader("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_7) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.91 Safari/534.30").
+		Get(url)
+
+	if err != nil {
+		return "", errors.Wrap(err, "could not perform request on "+url)
+	}
+	if resp.IsError() {
+		return "", &badRequest{Message: "could not perform request with " + url + " status code " + string(resp.StatusCode())}
+	}
+	return resp.String(), nil
+}
+
+type badRequest struct {
+	Message string `json:"message,omitempty"`
+}
+
+func (BadRequest *badRequest) Error() string {
+	return "Required request fields are not filled"
+}
--- a/vendor/github.com/advancedlogic/GoOse/images.go
+++ b/vendor/github.com/advancedlogic/GoOse/images.go
@@ -0,0 +1,328 @@
+package goose
+
+import (
+	"github.com/PuerkitoBio/goquery"
+	"net/url"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+type candidate struct {
+	url     string
+	surface int
+	score   int
+}
+
+func (c *candidate) GetUrl() string {
+	return c.url
+}
+
+var largebig = regexp.MustCompile("(large|big|full)")
+
+var classRules = map[*regexp.Regexp]int{
+	regexp.MustCompile("(promo|ads|banner)"): -1}
+
+var rules = map[*regexp.Regexp]int{
+	largebig:                                   1,
+	regexp.MustCompile("upload"):               1,
+	regexp.MustCompile("media"):                1,
+	regexp.MustCompile("gravatar.com"):         -1,
+	regexp.MustCompile("feeds.feedburner.com"): -1,
+	regexp.MustCompile("(?i)icon"):             -1,
+	regexp.MustCompile("(?i)logo"):             -1,
+	regexp.MustCompile("(?i)spinner"):          -1,
+	regexp.MustCompile("(?i)loading"):          -1,
+	regexp.MustCompile("(?i)ads"):              -1,
+	regexp.MustCompile("badge"):                -1,
+	regexp.MustCompile("1x1"):                  -1,
+	regexp.MustCompile("pixel"):                -1,
+	regexp.MustCompile("thumbnail[s]*"):        -1,
+	regexp.MustCompile(".html|" +
+		".gif|" +
+		".ico|" +
+		"button|" +
+		"twitter.jpg|" +
+		"facebook.jpg|" +
+		"ap_buy_photo|" +
+		"digg.jpg|" +
+		"digg.png|" +
+		"delicious.png|" +
+		"facebook.png|" +
+		"reddit.jpg|" +
+		"doubleclick|" +
+		"diggthis|" +
+		"diggThis|" +
+		"adserver|" +
+		"/(ads|promos|banners)/|" +
+		"ec.atdmt.com|" +
+		"mediaplex.com|" +
+		"adsatt|" +
+		"view.atdmt"): -1}
+
+func getImageSrc(tag *goquery.Selection) string {
+	src, _ := tag.Attr("src")
+	// skip inline images
+	if strings.Contains(src, "data:image/") {
+		src = ""
+	}
+	if src == "" {
+		src, _ = tag.Attr("data-src")
+	}
+	if src == "" {
+		src, _ = tag.Attr("data-lazy-src")
+	}
+	return src
+}
+
+func score(tag *goquery.Selection) int {
+	src := getImageSrc(tag)
+	if src == "" {
+		return -1
+	}
+	tagScore := 0
+	for rule, score := range rules {
+		if rule.MatchString(src) {
+			tagScore += score
+		}
+	}
+
+	alt, exists := tag.Attr("alt")
+	if exists {
+		if strings.Contains(alt, "thumbnail") {
+			tagScore--
+		}
+	}
+
+	id, exists := tag.Attr("id")
+	if exists {
+		if id == "fbPhotoImage" {
+			tagScore++
+		}
+	}
+
+	class, exists := tag.Attr("class")
+	if exists {
+		for rule, score := range classRules {
+			if rule.MatchString(class) {
+				tagScore += score
+			}
+		}
+	}
+
+	return tagScore
+}
+
+// WebPageImageResolver fetches all candidate images from the HTML page
+func WebPageImageResolver(doc *goquery.Document) ([]candidate, int) {
+	imgs := doc.Find("img")
+
+	var candidates []candidate
+	significantSurface := 320 * 200
+	significantSurfaceCount := 0
+	src := ""
+	imgs.Each(func(i int, tag *goquery.Selection) {
+		var surface int
+		src = getImageSrc(tag)
+		if src == "" {
+			return
+		}
+
+		width, _ := tag.Attr("width")
+		height, _ := tag.Attr("height")
+		if width != "" {
+			w, _ := strconv.Atoi(width)
+			if height != "" {
+				h, _ := strconv.Atoi(height)
+				surface = w * h
+			} else {
+				surface = w
+			}
+		} else {
+			if height != "" {
+				surface, _ = strconv.Atoi(height)
+			} else {
+				surface = 0
+			}
+		}
+
+		if surface > significantSurface {
+			significantSurfaceCount++
+		}
+
+		tagscore := score(tag)
+		if tagscore >= 0 {
+			c := candidate{
+				url:     src,
+				surface: surface,
+				score:   score(tag),
+			}
+			candidates = append(candidates, c)
+		}
+	})
+
+	if len(candidates) == 0 {
+		return nil, 0
+	}
+
+	return candidates, significantSurfaceCount
+
+}
+
+// WebPageResolver fetches the main image from the HTML page
+func WebPageResolver(article *Article) string {
+	candidates, significantSurfaceCount := WebPageImageResolver(article.Doc)
+	if candidates == nil {
+		return ""
+	}
+	var bestCandidate candidate
+	var topImage string
+	if significantSurfaceCount > 0 {
+		bestCandidate = findBestCandidateFromSurface(candidates)
+	} else {
+		bestCandidate = findBestCandidateFromScore(candidates)
+	}
+
+	topImage = bestCandidate.url
+	a, err := url.Parse(topImage)
+	if err != nil {
+		return topImage
+	}
+	finalURL, err := url.Parse(article.FinalURL)
+	if err != nil {
+		return topImage
+	}
+	b := finalURL.ResolveReference(a)
+	topImage = b.String()
+
+	return topImage
+}
+
+func findBestCandidateFromSurface(candidates []candidate) candidate {
+	max := 0
+	var bestCandidate candidate
+	for _, candidate := range candidates {
+		surface := candidate.surface
+		if surface >= max {
+			max = surface
+			bestCandidate = candidate
+		}
+	}
+
+	return bestCandidate
+}
+
+func findBestCandidateFromScore(candidates []candidate) candidate {
+	max := 0
+	var bestCandidate candidate
+	for _, candidate := range candidates {
+		score := candidate.score
+		if score >= max {
+			max = score
+			bestCandidate = candidate
+		}
+	}
+
+	return bestCandidate
+}
+
+type ogTag struct {
+	tpe       string
+	attribute string
+	name      string
+	value     string
+}
+
+var ogTags = [4]ogTag{
+	{
+		tpe:       "facebook",
+		attribute: "property",
+		name:      "og:image",
+		value:     "content",
+	},
+	{
+		tpe:       "facebook",
+		attribute: "rel",
+		name:      "image_src",
+		value:     "href",
+	},
+	{
+		tpe:       "twitter",
+		attribute: "name",
+		name:      "twitter:image",
+		value:     "value",
+	},
+	{
+		tpe:       "twitter",
+		attribute: "name",
+		name:      "twitter:image",
+		value:     "content",
+	},
+}
+
+type ogImage struct {
+	url   string
+	tpe   string
+	score int
+}
+
+// OpenGraphResolver return OpenGraph properties
+func OpenGraphResolver(doc *goquery.Document) string {
+	meta := doc.Find("meta")
+	links := doc.Find("link")
+	var topImage string
+	meta = meta.Union(links)
+	var ogImages []ogImage
+	meta.Each(func(i int, tag *goquery.Selection) {
+		for _, ogTag := range ogTags {
+			attr, exist := tag.Attr(ogTag.attribute)
+			value, vexist := tag.Attr(ogTag.value)
+			if exist && attr == ogTag.name && vexist {
+				ogImage := ogImage{
+					url:   value,
+					tpe:   ogTag.tpe,
+					score: 0,
+				}
+
+				ogImages = append(ogImages, ogImage)
+			}
+		}
+	})
+	if len(ogImages) == 0 {
+		return ""
+	}
+	if len(ogImages) == 1 {
+		topImage = ogImages[0].url
+		goto IMAGE_FINALIZE
+	}
+	for _, ogImage := range ogImages {
+		if largebig.MatchString(ogImage.url) {
+			ogImage.score++
+		}
+		if ogImage.tpe == "twitter" {
+			ogImage.score++
+		}
+	}
+	topImage = findBestImageFromScore(ogImages).url
+IMAGE_FINALIZE:
+	if !strings.HasPrefix(topImage, "http") {
+		topImage = "http://" + topImage
+	}
+
+	return topImage
+}
+
+// assume that len(ogImages)>=2
+func findBestImageFromScore(ogImages []ogImage) ogImage {
+	max := 0
+	bestOGImage := ogImages[0]
+	for _, ogImage := range ogImages[1:] {
+		score := ogImage.score
+		if score > max {
+			max = score
+			bestOGImage = ogImage
+		}
+	}
+
+	return bestOGImage
+}
--- a/vendor/github.com/advancedlogic/GoOse/outputformatter.go
+++ b/vendor/github.com/advancedlogic/GoOse/outputformatter.go
@@ -0,0 +1,180 @@
+package goose
+
+import (
+	"bytes"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+	"golang.org/x/net/html"
+)
+
+var normalizeWhitespaceRegexp = regexp.MustCompile(`[ \r\f\v\t]+`)
+var normalizeNl = regexp.MustCompile(`[\n]+`)
+var validURLRegex = regexp.MustCompile("^http[s]?://")
+
+type outputFormatter struct {
+	topNode  *goquery.Selection
+	config   Configuration
+	language string
+}
+
+func (formatter *outputFormatter) getLanguage(lang string) string {
+	if formatter.config.useMetaLanguage && "" != lang {
+		return lang
+	}
+	return formatter.config.targetLanguage
+}
+
+func (formatter *outputFormatter) getTopNode() *goquery.Selection {
+	return formatter.topNode
+}
+
+func (formatter *outputFormatter) getFormattedText(topNode *goquery.Selection, lang string) (output string, links []string) {
+	formatter.topNode = topNode
+	formatter.language = formatter.getLanguage(lang)
+	if formatter.language == "" {
+		formatter.language = formatter.config.targetLanguage
+	}
+	formatter.removeNegativescoresNodes()
+	links = formatter.linksToText()
+	formatter.replaceTagsWithText()
+	formatter.removeParagraphsWithFewWords()
+
+	output = formatter.getOutputText()
+	return output, links
+}
+
+func (formatter *outputFormatter) convertToText() string {
+	var txts []string
+	selections := formatter.topNode
+	selections.Each(func(i int, s *goquery.Selection) {
+		txt := s.Text()
+		if txt != "" {
+			// txt = txt //unescape
+			txtLis := strings.Trim(txt, "\n")
+			txts = append(txts, txtLis)
+		}
+	})
+	return strings.Join(txts, "\n\n")
+}
+
+// check if this is a valid URL
+func isValidURL(u string) bool {
+	return validURLRegex.MatchString(u)
+}
+
+func (formatter *outputFormatter) linksToText() []string {
+	var urlList []string
+	links := formatter.topNode.Find("a")
+	links.Each(func(i int, a *goquery.Selection) {
+		imgs := a.Find("img")
+		// ignore linked images
+		if imgs.Length() == 0 {
+			// save a list of URLs
+			url, _ := a.Attr("href")
+			if isValidURL(url) {
+				urlList = append(urlList, url)
+			}
+			// replace <a> tag with its text contents
+			replaceTagWithContents(a, whitelistedExtAtomTypes)
+
+			// see whether we can collapse the parent node now
+			replaceTagWithContents(a.Parent(), whitelistedTextAtomTypes)
+		}
+	})
+
+	return urlList
+}
+
+// Text gets the combined text contents of each element in the set of matched
+// elements, including their descendants.
+//
+// @see https://github.com/PuerkitoBio/goquery/blob/master/property.go
+func (formatter *outputFormatter) Text(s *goquery.Selection) string {
+	var buf bytes.Buffer
+
+	// Slightly optimized vs calling Each: no single selection object created
+	var f func(*html.Node)
+	f = func(n *html.Node) {
+		if n.Type == html.TextNode && 0 == n.DataAtom { // NB: had to add the DataAtom check to avoid printing text twice when a textual node embeds another textual node
+			// Keep newlines and spaces, like jQuery
+			buf.WriteString(n.Data)
+		}
+		if n.FirstChild != nil {
+			for c := n.FirstChild; c != nil; c = c.NextSibling {
+				f(c)
+			}
+		}
+	}
+	for _, n := range s.Nodes {
+		f(n)
+	}
+
+	return buf.String()
+}
+
+func (formatter *outputFormatter) getOutputText() string {
+	//out := formatter.topNode.Text()
+	out := formatter.Text(formatter.topNode)
+	out = normalizeWhitespaceRegexp.ReplaceAllString(out, " ")
+
+	strArr := strings.Split(out, "\n")
+	resArr := []string{}
+
+	for i, v := range strArr {
+		v = strings.TrimSpace(v)
+		if v != "" {
+			resArr = append(resArr, v)
+		} else if i > 2 && strArr[i-2] != "" {
+			resArr = append(resArr, "")
+		}
+	}
+
+	out = strings.Join(resArr, "\n")
+	out = normalizeNl.ReplaceAllString(out, "\n\n")
+
+	out = strings.TrimSpace(out)
+	return out
+}
+
+func (formatter *outputFormatter) removeNegativescoresNodes() {
+	gravityItems := formatter.topNode.Find("*[gravityScore]")
+	gravityItems.Each(func(i int, s *goquery.Selection) {
+		var score int
+		sscore, exists := s.Attr("gravityScore")
+		if exists {
+			score, _ = strconv.Atoi(sscore)
+			if score < 1 {
+				sNode := s.Get(0)
+				sNode.Parent.RemoveChild(sNode)
+			}
+		}
+
+	})
+}
+
+func (formatter *outputFormatter) replaceTagsWithText() {
+	for _, tag := range []string{"em", "strong", "b", "i", "span", "h1", "h2", "h3", "h4"} {
+		nodes := formatter.topNode.Find(tag)
+		nodes.Each(func(i int, node *goquery.Selection) {
+			replaceTagWithContents(node, whitelistedTextAtomTypes)
+		})
+	}
+}
+
+func (formatter *outputFormatter) removeParagraphsWithFewWords() {
+	language := formatter.language
+	if language == "" {
+		language = "en"
+	}
+	allNodes := formatter.topNode.Children()
+	allNodes.Each(func(i int, s *goquery.Selection) {
+		sw := formatter.config.stopWords.stopWordsCount(language, s.Text())
+		if sw.wordCount < 5 && s.Find("object").Length() == 0 && s.Find("em").Length() == 0 {
+			node := s.Get(0)
+			node.Parent.RemoveChild(node)
+		}
+	})
+}
--- a/vendor/github.com/advancedlogic/GoOse/parser.go
+++ b/vendor/github.com/advancedlogic/GoOse/parser.go
@@ -0,0 +1,90 @@
+package goose
+
+import (
+	"github.com/PuerkitoBio/goquery"
+	"golang.org/x/net/html"
+)
+
+// Parser is an HTML parser specialised in extraction of main content and other properties
+type Parser struct{}
+
+// NewParser returns an HTML parser
+func NewParser() *Parser {
+	return &Parser{}
+}
+
+func (p Parser) dropTag(selection *goquery.Selection) {
+	selection.Each(func(i int, s *goquery.Selection) {
+		replaceTagWithContents(s, whitelistedTextAtomTypes)
+	})
+}
+
+func (p Parser) indexOfAttribute(selection *goquery.Selection, attr string) int {
+	node := selection.Get(0)
+	for i, a := range node.Attr {
+		if a.Key == attr {
+			return i
+		}
+	}
+	return -1
+}
+
+func (p Parser) delAttr(selection *goquery.Selection, attr string) {
+	idx := p.indexOfAttribute(selection, attr)
+	if idx > -1 {
+		node := selection.Get(0)
+		node.Attr = append(node.Attr[:idx], node.Attr[idx+1:]...)
+	}
+}
+
+func (p Parser) getElementsByTags(div *goquery.Selection, tags []string) *goquery.Selection {
+	selection := new(goquery.Selection)
+	for _, tag := range tags {
+		selections := div.Find(tag)
+		if selections != nil {
+			selection = selection.Union(selections)
+		}
+	}
+	return selection
+}
+
+func (p Parser) clear(selection *goquery.Selection) {
+	selection.Nodes = make([]*html.Node, 0)
+}
+
+func (p Parser) removeNode(selection *goquery.Selection) {
+	if selection != nil {
+		node := selection.Get(0)
+		if node != nil && node.Parent != nil {
+			node.Parent.RemoveChild(node)
+		}
+	}
+}
+
+func (p Parser) name(selector string, selection *goquery.Selection) string {
+	value, exists := selection.Attr(selector)
+	if exists {
+		return value
+	}
+	return ""
+}
+
+func (p Parser) setAttr(selection *goquery.Selection, attr string, value string) {
+	if selection.Size() > 0 {
+		node := selection.Get(0)
+		var attrs []html.Attribute
+		for _, a := range node.Attr {
+			if a.Key != attr {
+				newAttr := new(html.Attribute)
+				newAttr.Key = a.Key
+				newAttr.Val = a.Val
+				attrs = append(attrs, *newAttr)
+			}
+		}
+		newAttr := new(html.Attribute)
+		newAttr.Key = attr
+		newAttr.Val = value
+		attrs = append(attrs, *newAttr)
+		node.Attr = attrs
+	}
+}
--- a/vendor/github.com/advancedlogic/GoOse/stopwords.go
+++ b/vendor/github.com/advancedlogic/GoOse/stopwords.go
--- a/vendor/github.com/advancedlogic/GoOse/videos.go
+++ b/vendor/github.com/advancedlogic/GoOse/videos.go
@@ -0,0 +1,165 @@
+package goose
+
+import (
+	"strconv"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/fatih/set"
+)
+
+// VideoExtractor can extract the main video from an HTML page
+type VideoExtractor struct {
+	article    *Article
+	config     Configuration
+	candidates *set.Set
+	movies     *set.Set
+}
+
+type video struct {
+	embedType string
+	provider  string
+	width     int
+	height    int
+	embedCode string
+	src       string
+}
+
+// NewVideoExtractor returns a new instance of a HTML video extractor
+func NewVideoExtractor() VideoExtractor {
+	return VideoExtractor{
+		candidates: set.New(set.ThreadSafe).(*set.Set),
+		movies:     set.New(set.ThreadSafe).(*set.Set),
+	}
+}
+
+var videoTags = [4]string{"iframe", "embed", "object", "video"}
+var videoProviders = [4]string{"youtube", "vimeo", "dailymotion", "kewego"}
+
+func (ve *VideoExtractor) getEmbedCode(node *goquery.Selection) string {
+	return node.Text()
+}
+
+func (ve *VideoExtractor) getWidth(node *goquery.Selection) int {
+	value, exists := node.Attr("width")
+	if exists {
+		nvalue, _ := strconv.Atoi(value)
+		return nvalue
+	}
+	return 0
+}
+
+func (ve *VideoExtractor) getHeight(node *goquery.Selection) int {
+	value, exists := node.Attr("height")
+	if exists {
+		nvalue, _ := strconv.Atoi(value)
+		return nvalue
+	}
+	return 0
+}
+
+func (ve *VideoExtractor) getSrc(node *goquery.Selection) string {
+	value, exists := node.Attr("src")
+	if exists {
+		return value
+	}
+	return ""
+}
+
+func (ve *VideoExtractor) getProvider(src string) string {
+	if src != "" {
+		for _, provider := range videoProviders {
+			if strings.Contains(src, provider) {
+				return provider
+			}
+		}
+	}
+	return ""
+}
+
+func (ve *VideoExtractor) getVideo(node *goquery.Selection) video {
+	src := ve.getSrc(node)
+	video := video{
+		embedCode: ve.getEmbedCode(node),
+		embedType: node.Get(0).DataAtom.String(),
+		width:     ve.getWidth(node),
+		height:    ve.getHeight(node),
+		src:       src,
+		provider:  ve.getProvider(src),
+	}
+	return video
+}
+
+func (ve *VideoExtractor) getIFrame(node *goquery.Selection) video {
+	return ve.getVideo(node)
+}
+
+func (ve *VideoExtractor) getVideoTag(node *goquery.Selection) video {
+	return video{}
+}
+
+func (ve *VideoExtractor) getEmbedTag(node *goquery.Selection) video {
+	parent := node.Parent()
+	if parent != nil {
+		parentTag := parent.Get(0).DataAtom.String()
+		if parentTag == "object" {
+			return ve.getObjectTag(node)
+		}
+	}
+	return ve.getVideo(node)
+}
+
+func (ve *VideoExtractor) getObjectTag(node *goquery.Selection) video {
+	childEmbedTag := node.Find("embed")
+	if ve.candidates.Has(childEmbedTag) {
+		ve.candidates.Remove(childEmbedTag)
+	}
+	srcNode := node.Find(`param[name="movie"]`)
+	if srcNode == nil || srcNode.Length() == 0 {
+		return video{}
+	}
+
+	src, _ := srcNode.Attr("value")
+	provider := ve.getProvider(src)
+	if provider == "" {
+		return video{}
+	}
+	video := ve.getVideo(node)
+	video.provider = provider
+	video.src = src
+	return video
+}
+
+// GetVideos returns the video tags embedded in the article
+func (ve *VideoExtractor) GetVideos(doc *goquery.Document) *set.Set {
+	var nodes *goquery.Selection
+	for _, videoTag := range videoTags {
+		tmpNodes := doc.Find(videoTag)
+		if nodes == nil {
+			nodes = tmpNodes
+		} else {
+			nodes.Union(tmpNodes)
+		}
+	}
+
+	nodes.Each(func(i int, node *goquery.Selection) {
+		tag := node.Get(0).DataAtom.String()
+		var movie video
+		switch tag {
+		case "video":
+			movie = ve.getVideoTag(node)
+		case "embed":
+			movie = ve.getEmbedTag(node)
+		case "object":
+			movie = ve.getObjectTag(node)
+		case "iframe":
+			movie = ve.getIFrame(node)
+		}
+
+		if movie.src != "" {
+			ve.movies.Add(movie)
+		}
+	})
+
+	return ve.movies
+}
--- a/vendor/github.com/advancedlogic/GoOse/wordstats.go
+++ b/vendor/github.com/advancedlogic/GoOse/wordstats.go
@@ -0,0 +1,39 @@
+package goose
+
+import (
+	"github.com/fatih/set"
+)
+
+//some word statistics
+type wordStats struct {
+	//total number of stopwords or good words that we can calculate
+	stopWordCount int
+	//total number of words on a node
+	wordCount int
+	//holds an actual list of the stop words we found
+	stopWords *set.Set
+}
+
+func (w *wordStats) getStopWords() *set.Set {
+	return w.stopWords
+}
+
+func (w *wordStats) setStopWords(stopWords *set.Set) {
+	w.stopWords = stopWords
+}
+
+func (w *wordStats) getStopWordCount() int {
+	return w.stopWordCount
+}
+
+func (w *wordStats) setStopWordCount(stopWordCount int) {
+	w.stopWordCount = stopWordCount
+}
+
+func (w *wordStats) getWordCount() int {
+	return w.wordCount
+}
+
+func (w *wordStats) setWordCount(wordCount int) {
+	w.wordCount = wordCount
+}