8d5be2d657
* Document extractor service * Fixing vendor modules * Addressing PR Review comments * Some small simplifications * Fixing a linter complain * simplifying a bit the code using package variables Co-authored-by: Mattermod <mattermod@users.noreply.github.com>
32 строки
635 B
Go
32 строки
635 B
Go
package docconv
|
|
|
|
import (
|
|
"bytes"
|
|
"io"
|
|
|
|
"github.com/advancedlogic/GoOse"
|
|
)
|
|
|
|
// ConvertURL fetches the HTML page at the URL given in the io.Reader.
|
|
func ConvertURL(input io.Reader, readability bool) (string, map[string]string, error) {
|
|
meta := make(map[string]string)
|
|
|
|
buf := new(bytes.Buffer)
|
|
_, err := buf.ReadFrom(input)
|
|
if err != nil {
|
|
return "", nil, err
|
|
}
|
|
|
|
g := goose.New()
|
|
article, err := g.ExtractFromURL(buf.String())
|
|
if err != nil {
|
|
return "", nil, err
|
|
}
|
|
|
|
meta["title"] = article.Title
|
|
meta["description"] = article.MetaDescription
|
|
meta["image"] = article.TopImage
|
|
|
|
return article.CleanedText, meta, nil
|
|
}
|