Document extractor service (#15665)
* Document extractor service * Fixing vendor modules * Addressing PR Review comments * Some small simplifications * Fixing a linter complain * simplifying a bit the code using package variables Co-authored-by: Mattermod <mattermod@users.noreply.github.com>
Этот коммит содержится в:
Jesús Espino
коммит произвёл
GitHub
GitHub
родитель
04ef5c682e
Коммит
8d5be2d657
32
vendor/github.com/JalfResi/justext/.gitignore
сгенерированный
поставляемый
Обычный файл
32
vendor/github.com/JalfResi/justext/.gitignore
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,32 @@
|
||||
|
||||
src/_go_.6
|
||||
|
||||
src/_obj/ourscienceistight/gojustext.a
|
||||
|
||||
src/example/_go_.6
|
||||
|
||||
src/example/gojustext
|
||||
|
||||
src/main.go_old
|
||||
|
||||
src/.DS_Store
|
||||
|
||||
src/stoplists/.DS_Store
|
||||
|
||||
src/example/*.html
|
||||
|
||||
src/example/*.htm
|
||||
|
||||
src/stoplists/*.txt
|
||||
|
||||
src/example/hp-envy-14-spectre-review
|
||||
|
||||
src/example/t.txt
|
||||
|
||||
src/example/t2.txt
|
||||
|
||||
gojustext
|
||||
|
||||
gojustext.sublime-project
|
||||
|
||||
gojustext.sublime-workspace
|
||||
48
vendor/github.com/JalfResi/justext/README.md
сгенерированный
поставляемый
Обычный файл
48
vendor/github.com/JalfResi/justext/README.md
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,48 @@
|
||||
justext
|
||||
=======
|
||||
|
||||
A Go package that implements the JusText boilerplate removal algorithm (http://code.google.com/p/justext/)
|
||||
|
||||
## Install
|
||||
|
||||
go get github.com/JalfResi/justext
|
||||
|
||||
And import:
|
||||
|
||||
import "github.com/JalfResi/justext"
|
||||
|
||||
## Usage
|
||||
|
||||
Supports all stoplist files available at http://code.google.com/p/justext/source/browse/#svn%2Ftrunk%2Fjustext%2Fstoplists
|
||||
|
||||
Justext expects valid HTML; it is your responsability to ensure that valid HTML is passed to Justext. To make things easier
|
||||
I have written a CGO wrapper around libtidy which you can find here: [github.com/JalfResi/GoTidy](https://github.com/JalfResi/GoTidy)
|
||||
In the future, once exp/html is part of the standard packages I will refactor JusText to accept only valid HTML documents/strings.
|
||||
|
||||
Justext use the reader-writer idiom, alowing you to setup the reader with a common configuration and just pump out
|
||||
articles to the writer.
|
||||
|
||||
Example usage:
|
||||
|
||||
// Create a justext reader from another reader
|
||||
reader := justext.NewReader(os.Stdin)
|
||||
|
||||
// Configure the reader
|
||||
reader.LengthLow = 70
|
||||
reader.LengthHigh = 200
|
||||
reader.Stoplist = stoplist // The stoplist map[string]bool
|
||||
reader.StopwordsLow = 0.3
|
||||
reader.StopwordsHigh = 0.32
|
||||
reader.MaxLinkDensity = 0.2
|
||||
reader.MaxHeadingDistance = 200
|
||||
reader.NoHeadings = false
|
||||
|
||||
// Read from the reader to generate a paragraph set
|
||||
paragraphSet, _ := reader.ReadAll()
|
||||
|
||||
// Create a writer from another writer
|
||||
writer := justext.NewWriter(os.Stdout)
|
||||
// Write the paragraph set to the writer
|
||||
writer.WriteAll(paragraphSet)
|
||||
|
||||
|
||||
3
vendor/github.com/JalfResi/justext/TODO.txt
сгенерированный
поставляемый
Обычный файл
3
vendor/github.com/JalfResi/justext/TODO.txt
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,3 @@
|
||||
TODO
|
||||
====
|
||||
- Stoplists need to be separtae subpackages (use the init() package method of registration)
|
||||
66
vendor/github.com/JalfResi/justext/classifyParagraphs.go
сгенерированный
поставляемый
Обычный файл
66
vendor/github.com/JalfResi/justext/classifyParagraphs.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,66 @@
|
||||
package justext
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var findHeadings *regexp.Regexp = regexp.MustCompile("(^h[123456]|.h[123456])")
|
||||
var copyrightChar *regexp.Regexp = regexp.MustCompile("(\u0161|©)")
|
||||
var findSelect *regexp.Regexp = regexp.MustCompile("(^select|.select)")
|
||||
|
||||
func classifyParagraphs(paragraphs []*Paragraph, stoplist map[string]bool, lengthLow int, lengthHigh int, stopwordsLow float64, stopwordsHigh float64, maxLinkDensity float64, noHeadings bool) {
|
||||
for _, paragraph := range paragraphs {
|
||||
var length int = len(paragraph.Text)
|
||||
var stopwordCount int = 0
|
||||
for _, word := range strings.Split(paragraph.Text, " ") {
|
||||
if _, ok := stoplist[word]; ok {
|
||||
stopwordCount += 1
|
||||
}
|
||||
}
|
||||
|
||||
var stopwordDensity float64 = 0.0
|
||||
var linkDensity float64 = 0.0
|
||||
var wordCount int = paragraph.WordCount
|
||||
|
||||
if wordCount > 0 {
|
||||
stopwordDensity = 1.0 * float64(stopwordCount) / float64(wordCount)
|
||||
linkDensity = float64(paragraph.LinkedCharCount) / float64(length)
|
||||
}
|
||||
|
||||
paragraph.StopwordCount = stopwordCount
|
||||
paragraph.StopwordDensity = stopwordDensity
|
||||
paragraph.LinkDensity = linkDensity
|
||||
paragraph.Heading = bool(!noHeadings && findHeadings.MatchString(paragraph.DomPath))
|
||||
|
||||
if linkDensity > maxLinkDensity {
|
||||
paragraph.CfClass = "bad"
|
||||
} else if copyrightChar.MatchString(paragraph.Text) {
|
||||
paragraph.CfClass = "bad"
|
||||
} else if findSelect.MatchString(paragraph.DomPath) {
|
||||
paragraph.CfClass = "bad"
|
||||
} else {
|
||||
if length < lengthLow {
|
||||
if paragraph.LinkedCharCount > 0 {
|
||||
paragraph.CfClass = "bad"
|
||||
} else {
|
||||
paragraph.CfClass = "short"
|
||||
}
|
||||
} else {
|
||||
if stopwordDensity >= stopwordsHigh {
|
||||
if length > lengthHigh {
|
||||
paragraph.CfClass = "good"
|
||||
} else {
|
||||
paragraph.CfClass = "neargood"
|
||||
}
|
||||
} else {
|
||||
if stopwordDensity >= stopwordsLow {
|
||||
paragraph.CfClass = "neargood"
|
||||
} else {
|
||||
paragraph.CfClass = "bad"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
50
vendor/github.com/JalfResi/justext/defaultTemplate.go
сгенерированный
поставляемый
Обычный файл
50
vendor/github.com/JalfResi/justext/defaultTemplate.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,50 @@
|
||||
package justext
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"io"
|
||||
"reflect"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
var _DefaultTemplate = "" +
|
||||
"\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\x74\x90\x3d\x6e\xc4\x20" +
|
||||
"\x10\x46\x6b\xfb\x14\xc8\x4a\x8d\xfb\x08\x53\x24\x45\x92\x26\x8a" +
|
||||
"\x14\x5f\x60\x6c\x88\x41\xc2\x80\x80\x22\x2b\xc4\xdd\x17\xc3\x7a" +
|
||||
"\xff\xb4\x5b\x81\xde\x3c\xe6\x1b\x86\x88\xb0\x2a\xda\x92\xc9\xb0" +
|
||||
"\x03\x6d\x63\x7c\xd1\xe6\xcd\x48\xc5\x9d\x55\x10\x38\x7a\x1d\x10" +
|
||||
"\xfe\xbe\x26\x29\x65\xc9\x81\x5e\x38\xc2\x3f\xe0\x60\x71\x60\x85" +
|
||||
"\xcf\xb4\x89\x51\xfe\xa1\x2f\xff\x61\x0c\x43\xf8\x5d\x81\x2f\xb4" +
|
||||
"\x62\xfc\xc9\x81\x49\xbd\x14\xd2\x10\x41\x63\x1c\x9d\x5c\x7f\x2d" +
|
||||
"\xcc\xb9\xcf\xc8\xff\x43\x4a\xa4\x17\xb4\xf8\x5c\x79\x7e\x12\xed" +
|
||||
"\x63\xd1\xd2\x5c\xac\xae\x66\x35\xfb\xfc\xa8\xe4\x69\x13\xd0\xed" +
|
||||
"\x4f\xf6\x86\x68\xde\x26\x1b\xba\xe9\x52\xea\x9e\x85\xdc\x25\x94" +
|
||||
"\xcb\x7e\x92\xbe\x6e\x2c\x4f\xbd\x2d\xf0\x18\x00\x00\xff\xff\x2c" +
|
||||
"\xc5\xf5\x5d\x47\x01\x00\x00"
|
||||
|
||||
// DefaultTemplate returns the binary data for a given file.
|
||||
func DefaultTemplate() []byte {
|
||||
// This bit of black magic ensures we do not get
|
||||
// unneccesary memcpy's and can read directly from
|
||||
// the .rodata section.
|
||||
var empty [0]byte
|
||||
sx := (*reflect.StringHeader)(unsafe.Pointer(&_DefaultTemplate))
|
||||
b := empty[:]
|
||||
bx := (*reflect.SliceHeader)(unsafe.Pointer(&b))
|
||||
bx.Data = sx.Data
|
||||
bx.Len = len(_DefaultTemplate)
|
||||
bx.Cap = bx.Len
|
||||
|
||||
gz, err := gzip.NewReader(bytes.NewBuffer(b))
|
||||
|
||||
if err != nil {
|
||||
panic("Decompression failed: " + err.Error())
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
io.Copy(&buf, gz)
|
||||
gz.Close()
|
||||
|
||||
return buf.Bytes()
|
||||
}
|
||||
181
vendor/github.com/JalfResi/justext/detailedTemplate.go
сгенерированный
поставляемый
Обычный файл
181
vendor/github.com/JalfResi/justext/detailedTemplate.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,181 @@
|
||||
package justext
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"io"
|
||||
"reflect"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
var _DetailedTemplate = "" +
|
||||
"\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\xe4\x59\x59\x73\xe2\x3a" +
|
||||
"\xf6\x7f\x4e\x3e\x85\xff\xfc\x6f\x4d\x27\x45\x9a\x1d\x02\x09\x64" +
|
||||
"\xc6\x18\x13\x76\x02\x06\x02\x99\x9a\xea\x12\xb6\x6c\x14\xaf\xb1" +
|
||||
"\x65\x8c\xc9\xf0\xdd\x47\x32\x66\x0b\x24\x4d\xcf\x43\xdf\x5b\x35" +
|
||||
"\xbc\xd8\x3e\x3a\xfb\xf9\x49\x3a\x12\xc5\x19\xd6\xb5\x87\xcb\xa2" +
|
||||
"\x83\x7d\x0d\x3e\x5c\x5e\xfc\x03\xe9\x96\x69\x63\xc6\xb5\xb5\xab" +
|
||||
"\x19\xc6\xd6\x5d\x3c\x2e\x9b\x06\x76\x62\x8a\x69\x2a\x1a\x04\x16" +
|
||||
"\x72\x62\xa2\xa9\xc7\x45\xc7\xf9\xbb\x0c\x74\xa4\xf9\xa5\xae\x05" +
|
||||
"\x8d\xa8\x00\x0c\xe7\x2e\x93\x48\xdc\xa4\x13\x89\xbf\x39\xee\xd4" +
|
||||
"\x81\xb8\xa4\x01\x8c\x8c\x1b\xd1\xb7\x91\xa6\x21\xf1\x3b\x5c\xe0" +
|
||||
"\xed\xc7\x8d\x62\x43\xa8\x06\xa4\x39\x82\xd8\x00\x3a\x74\xe0\x4d" +
|
||||
"\x20\x40\x89\xd7\xf7\x97\x17\x97\x17\x53\x53\xf2\x2f\x2f\xde\x2f" +
|
||||
"\x2f\x2e\xa8\x0b\xdf\xd7\xe6\xee\x98\x6f\xd4\x20\x43\x0d\x7e\xbb" +
|
||||
"\x61\x1c\xf2\xf8\xee\x40\x1b\xc9\xf7\x1b\x3e\x0f\x22\x65\x86\xef" +
|
||||
"\x18\xe2\xc9\x3d\x43\x88\x3a\xb0\x15\xa2\x56\x83\x32\x21\x02\x17" +
|
||||
"\x9b\xf7\x3b\xa2\xbd\x66\xdd\x50\x3d\x24\xe1\xd9\x1d\x93\x4f\x24" +
|
||||
"\xac\x05\xf9\x5e\x5d\x5e\x5e\x38\x16\x30\x62\x0e\x36\x2d\xcf\xb4" +
|
||||
"\x25\x86\x3a\x83\x89\x83\xdf\x25\x28\x9a\x36\x71\xd7\x34\xee\x18" +
|
||||
"\xd7\x90\xa0\xad\x21\x03\x86\x22\x16\x4d\x96\x74\xc3\x58\xb1\x19" +
|
||||
"\x04\x12\x32\x14\xfa\x3a\x05\x81\x34\x43\x7e\xa1\x95\x6c\x76\x6d" +
|
||||
"\x85\x92\x02\x9d\x40\x43\x0a\x51\xf7\xea\x3a\x18\xc9\x7e\x38\x12" +
|
||||
"\x3a\x4a\x1c\xb8\x63\x52\x5b\xfe\x90\x3a\x35\x31\x36\xf5\xfd\x01" +
|
||||
"\x0b\x48\xd4\x62\x18\x6d\xfa\x88\x1e\x06\x9c\xde\xc6\x77\xe4\xec" +
|
||||
"\xc6\xcb\x29\x10\x55\xc5\x36\x49\x70\xdf\x45\x53\x33\xed\x3b\xe6" +
|
||||
"\xff\x45\x59\xdc\x4a\xed\x71\x1f\xa6\x3d\x13\xa6\x3d\x20\x3a\x68" +
|
||||
"\x09\xef\x98\x64\x2c\x0d\xf5\xad\xe4\x5e\x26\x4e\xd8\x90\x25\x29" +
|
||||
"\xe4\x94\xd0\x3c\x66\x01\x1b\x28\x36\xb0\x66\x3f\x24\x88\x01\xd2" +
|
||||
"\x9c\x0f\x39\x4c\x65\x76\x39\x94\x35\x13\x10\xfb\x41\x84\x21\x49" +
|
||||
"\x42\x8e\xa5\x01\x82\x19\xc3\xdc\x56\xe7\xb4\x5a\x0c\xa6\x1a\xdc" +
|
||||
"\xba\x45\x4a\x0d\x6d\xea\x92\x06\x2c\x87\x04\xb0\x79\x3b\x43\x05" +
|
||||
"\xde\x06\x77\x58\x8a\xe3\x12\x85\xa5\xd8\x0d\xac\xcd\x92\x74\x59" +
|
||||
"\x0b\xc6\x31\x35\x24\x91\x6c\xc8\xf2\x57\x36\x83\xda\xad\x0d\xdf" +
|
||||
"\x7c\xc6\xb2\x29\xd4\x89\x08\x0f\x4c\x15\xe4\xc2\x97\xa6\x68\xd9" +
|
||||
"\x7e\xa6\x43\x06\xe0\x4c\x77\x19\x6c\x9f\xe9\x31\xb6\xbf\x40\xcb" +
|
||||
"\xf9\x4e\x7f\xa9\x66\xe7\x77\x0c\xfa\xd0\x24\xeb\x4b\xb8\xee\xac" +
|
||||
"\x21\x45\x2b\x78\xff\x71\x25\xf9\x9e\x24\x40\xdf\x11\x83\x09\xba" +
|
||||
"\x9e\x56\x17\x5b\xd8\x4d\x35\x53\x54\xf7\x16\x96\x74\x72\xcd\x30" +
|
||||
"\x0b\xe7\x4a\x2a\xb3\xfe\xde\x73\xc9\x86\x16\xa4\x36\x0d\x33\x7c" +
|
||||
"\xfd\x30\x8e\x74\xa0\x10\x44\xd2\xf5\x59\x02\x18\xdc\x05\xdf\x71" +
|
||||
"\xcb\x50\xee\xa7\xc0\x81\xb9\xcc\x0d\x1a\x95\xbb\x7d\x2f\xd1\x7c" +
|
||||
"\x54\x4c\x96\xfc\x3a\xc2\x70\xc6\x0f\x15\xf2\x56\xce\xd3\xef\x3e" +
|
||||
"\xc7\x4e\xe8\x53\xb3\x6a\x52\x83\x52\x1b\x09\x8d\xef\x8d\xfa\x99" +
|
||||
"\xd4\x5b\x6a\x34\xee\xa5\x2b\x6d\xd6\xe7\x17\x3e\xdb\xe3\xde\xf8" +
|
||||
"\x1e\xab\xf3\xc1\x93\x43\x7c\xaf\x8c\xc2\x77\x87\xef\xb5\x5f\x0b" +
|
||||
"\x62\xa3\x8d\x9b\x53\x0c\x52\x42\x27\x5a\x49\xe4\x07\xc6\xb8\x76" +
|
||||
"\x9b\x03\xcd\x54\x35\x31\x9d\x98\xcf\x03\x55\x74\xb3\xf5\x45\x4b" +
|
||||
"\x5d\xf0\x6d\xae\x96\xb5\xa6\x3d\x76\x24\x69\x28\xa3\x2d\x13\xc3" +
|
||||
"\x7a\xaf\xde\x86\xd9\xa7\xbe\xe8\xf3\x6d\x1f\xa4\x9f\xd2\xec\x5b" +
|
||||
"\x53\x10\xdc\xe9\xe8\x09\x18\x8b\xaa\x66\xc1\x16\x18\xa3\x7e\x79" +
|
||||
"\xf8\x72\xcb\xab\x13\x7b\xde\x19\xbd\xb1\xd5\x41\x6e\x22\x8d\x27" +
|
||||
"\xed\xc9\x68\x24\x24\xeb\x8f\x2f\xc9\x49\x33\xca\xc7\x1f\xcb\xe2" +
|
||||
"\x04\xd5\x52\xb5\xea\xf8\x35\x5b\xad\x8d\xbb\x99\x6a\x66\x91\x6b" +
|
||||
"\x45\x93\xaf\xb7\xac\x50\x43\x9e\x33\x28\x3c\xb5\x66\xfd\xf9\x4b" +
|
||||
"\x27\x6a\x44\x0b\x2e\x1a\x62\x81\xef\xcb\x4e\x74\xdc\x99\xb0\x9c" +
|
||||
"\x97\xcc\x0f\xa7\xdd\x64\xe7\xcd\x86\x6a\xf2\x79\x6e\xcd\xa3\x65" +
|
||||
"\xf1\x55\x4d\x34\x52\x95\xd6\x44\x28\x24\x6a\xe5\x6a\x05\x60\x2e" +
|
||||
"\xaf\xf8\x8b\xac\x06\xe2\x51\x3b\x5d\x4b\xb3\x03\xbe\xda\x7d\x92" +
|
||||
"\x2b\xad\xaa\xfb\xda\xcd\x65\xdc\xe6\xa0\xba\x30\x3b\xe9\x27\xcb" +
|
||||
"\xc8\x38\x6f\x78\xc9\xdd\xbe\xe9\xa3\x7c\x42\x97\x04\x77\xe0\x4d" +
|
||||
"\xbb\x59\x3d\xe5\x8e\x86\x80\xeb\x75\xc1\x70\x2c\x88\x42\x83\x2b" +
|
||||
"\x14\x16\x8e\x5d\x90\x0a\xf5\x42\x2a\xee\x2f\x9b\x8b\x81\xee\xb9" +
|
||||
"\x43\x0f\x3e\xaa\xb5\x78\x5a\x2d\x64\x66\x15\x29\xe5\x2f\xec\x11" +
|
||||
"\x6c\x69\x62\xa7\xc7\x06\xa5\xe9\x0f\xb3\xbc\xad\x36\x14\x45\x29" +
|
||||
"\x95\xae\xf7\x10\x2a\x6a\xa6\x03\xa5\xf3\x31\xfa\x57\x06\xe4\x24" +
|
||||
"\x04\x24\xf7\xe6\x83\x32\xa5\x4e\xf9\x10\x90\x76\xea\xb9\x22\xa6" +
|
||||
"\xf9\x3a\xeb\x3f\x2e\x3d\x5e\x68\xe8\xbc\xa0\xd6\x39\x5e\x18\xb2" +
|
||||
"\x1c\x6f\xb1\x1e\x37\x44\x93\x19\xaf\x4f\xca\xdc\xc0\x01\x8b\x2e" +
|
||||
"\x28\x74\x32\x6c\x34\x8e\x53\x8d\xc7\xbc\x5b\x4e\xfa\xb2\x66\xbd" +
|
||||
"\xf8\x46\x3a\x2d\xc7\x93\xc3\x44\xed\x51\x9d\x73\x76\xaa\x30\xcd" +
|
||||
"\xb3\xe3\x72\xbf\x10\x2d\xeb\x70\xc2\xa2\x42\x2d\x9a\xcb\x03\x9c" +
|
||||
"\xeb\xb3\x48\xf7\xf3\xcd\xc6\x30\xdb\xcf\x9a\x23\xb9\x65\x56\xfb" +
|
||||
"\x39\x76\x32\x07\xe5\x7c\xc3\x7a\x75\xf9\xa1\x1d\x9d\x22\x24\xa4" +
|
||||
"\x9b\x29\xce\x8c\x3e\xce\xbd\x6a\x45\x89\xf2\x26\xa8\x44\xf9\xc4" +
|
||||
"\xb2\xae\xbe\x2e\x15\xd9\x06\x99\xda\xeb\x74\xb1\xe4\xf8\xe9\x22" +
|
||||
"\xa5\xd4\xb3\xea\x20\xd1\x2b\xa8\xcf\x0d\xdb\xee\x8d\x9e\x0c\x61" +
|
||||
"\xf6\xfa\xe2\xc5\xdd\x4e\x4e\xe8\xcc\xf3\x8e\xe1\xd7\x6e\xcb\x09" +
|
||||
"\xa9\x2b\xd8\x04\x65\xec\x52\xe8\x69\xfa\x14\x5a\x4b\x47\xf4\xa2" +
|
||||
"\x65\xde\x29\xf4\x58\xac\x76\x9f\x47\x39\xb8\x18\xc1\xf2\x4b\x56" +
|
||||
"\xab\x22\xfc\xc8\xdb\xb9\x96\xd9\x83\x75\xa1\x3e\x00\xa3\x51\xc3" +
|
||||
"\x18\x71\xee\x4b\xed\x16\x4b\x2d\x6d\xda\xec\xea\x6d\x37\x3a\x98" +
|
||||
"\xf5\x75\x1b\x8f\x5b\x5a\xd9\xd1\x0c\x33\xda\x28\x2c\x5e\x26\x96" +
|
||||
"\xe8\x77\xd9\xbe\xef\x67\xd8\x9a\xb1\x48\x35\xe7\xb3\xee\x5b\xbd" +
|
||||
"\x62\xa9\xac\x17\xf7\xf8\x0a\x28\x24\x97\x5c\x62\x32\x6e\xb6\x9a" +
|
||||
"\xaa\x69\xf0\x60\xd4\x66\x5f\x8c\x4e\x54\xc9\x65\x35\x38\xcb\x3e" +
|
||||
"\xd5\x13\x4e\x35\xa9\x3b\x8f\xd6\x78\x1e\x97\x85\x6e\xf6\x6d\x7a" +
|
||||
"\x2b\x4d\x92\xfd\xf4\xab\xe5\x56\xd5\x4c\x6b\x68\xd8\x05\x2b\x21" +
|
||||
"\xb6\xf2\x05\x2e\xeb\x95\x73\x95\x44\x66\x6e\xd4\xe1\x30\xd9\xd2" +
|
||||
"\x59\xcf\x91\xd5\xb6\xe4\x55\xab\xfd\x84\x37\x1f\xa6\x33\x12\x1a" +
|
||||
"\xa6\x9b\xf1\xd9\xf3\xb8\xde\x02\xd1\x38\x07\x86\xcb\x0c\x8e\xa3" +
|
||||
"\x46\x5c\xa9\xbc\xc8\x5a\x5a\x2f\xe4\xc4\x71\xdb\xfb\x0c\xec\xc5" +
|
||||
"\x78\xd8\x96\x16\x1d\xd1\x46\x16\x26\xfd\xa9\xec\x1a\x22\xed\xba" +
|
||||
"\x18\x67\x66\x7a\x3f\x80\x34\x07\x86\x08\xa5\xab\xeb\xcd\xe2\xbe" +
|
||||
"\xa1\xfc\x30\x2d\xca\xe6\x30\x25\x46\x32\x45\x57\x87\x06\x8e\x29" +
|
||||
"\x10\xf3\x1a\xa4\xaf\x65\xbf\x2e\x5d\x45\x3e\xf2\x46\xae\xef\x4f" +
|
||||
"\x2b\x89\x05\x6e\xc4\xc2\x59\x44\x54\x46\x82\x79\x14\x09\xd9\x0f" +
|
||||
"\x5c\xf9\x41\x1a\x41\xf5\x2b\xab\xc7\xdc\x5b\xbb\xc7\x43\xc7\x96" +
|
||||
"\x69\x27\xb3\x31\x3c\x43\x12\x3c\xdf\xf0\x31\xf7\xd6\xf0\xf1\xd0" +
|
||||
"\xb1\xe1\x48\xb8\xfe\x6c\x2b\x70\x20\xf4\xdb\x2b\xb0\x9f\x87\xdf" +
|
||||
"\x5a\x80\x3f\x25\xf9\x9b\x68\x0f\x0a\x80\x4d\x85\x1c\xc9\x7e\x4c" +
|
||||
"\x4d\xa4\x41\x9b\x70\x62\xf2\xee\x92\x03\x81\xb1\x2e\xc6\xc5\x7a" +
|
||||
"\xbc\xbc\x1b\xfe\xca\xc1\x23\xe6\xc0\xbf\x0b\x24\x33\x57\x47\x43" +
|
||||
"\x31\x51\x03\x8e\xd3\x21\x87\xb6\x18\x22\x67\x9f\x45\x57\xbe\x8a" +
|
||||
"\x6c\x77\xa5\xc8\x35\xf3\x7f\x25\xb2\xf3\xac\x7d\x38\x76\x62\x27" +
|
||||
"\x4c\x03\x0b\xdb\x2d\x1a\x1b\xd9\x85\x68\x26\xf6\xc2\xb9\x0a\x5c" +
|
||||
"\x58\x31\x50\x73\xe0\xb9\xda\x42\x27\x02\x7d\x41\x39\x8f\xf5\x9d" +
|
||||
"\xc2\xf1\x01\xd7\xae\x83\x0f\xfb\xc9\x4f\x40\xec\x94\xfd\x01\x50" +
|
||||
"\xa8\xf1\xab\x88\xb5\xad\xa7\x6c\xda\xcc\xd5\x1c\xd8\x0c\x2a\x91" +
|
||||
"\x93\x10\x2a\xee\xb4\xc4\x34\x68\x28\x78\x46\x88\xd1\xe8\xd6\x08" +
|
||||
"\xfd\x51\xee\x2d\x1b\xb1\xb5\x13\xf9\x27\xfa\xd7\xfd\x8e\x8f\x56" +
|
||||
"\x63\x3b\x76\xaa\x0a\xa4\xdf\x3d\xc8\x3f\xb3\xf7\xdb\x09\x7e\x39" +
|
||||
"\x99\xe8\x6f\xb5\x7e\x5d\xed\x01\x74\x1f\x65\x67\x81\xfd\xa3\xc0" +
|
||||
"\x21\xde\x3f\x8e\x9e\x31\xc1\x7f\xc5\x83\x93\x02\x87\xd3\xfc\xe7" +
|
||||
"\x1e\x1c\x4d\xb8\x63\x3c\xfd\xcf\x21\xe5\x73\x94\xfc\x39\x35\x3a" +
|
||||
"\xda\x0e\x7f\x3b\x4e\x4f\xa3\xe4\xe8\x28\xba\xab\xc6\x0d\x63\x49" +
|
||||
"\x48\x3a\x86\xce\xf6\x22\xe1\x73\x8f\x03\xc1\xfb\x4f\xe4\x3e\xed" +
|
||||
"\x52\x98\xb5\xc0\x2f\x41\x62\x57\xe5\x8f\x60\xd8\xb5\xfe\x1c\x3d" +
|
||||
"\x3d\x53\x3b\xf4\xfc\xbc\x29\x02\x5d\xab\x7f\x4d\x98\x9c\xe1\x4f" +
|
||||
"\xf7\x16\x7f\x89\x1c\xae\x01\xf6\x1b\x52\x28\x49\xff\x7d\x0a\x45" +
|
||||
"\x59\x8c\x6c\x3a\xe6\xb0\x53\x2e\xd2\xdb\x53\xd2\x30\x17\x25\x34" +
|
||||
"\x67\x90\x54\x3a\xb1\xc1\x33\x41\x00\xa5\xbd\x5d\x93\x31\x0d\x51" +
|
||||
"\x43\xa2\xba\xe1\x3e\xd9\x5b\x44\x1e\x8a\x71\xa2\xf4\xe1\xf2\xfd" +
|
||||
"\xdd\x06\x86\x02\x99\x3f\x82\xf0\x6f\x98\x3f\x2c\xe6\xae\xc4\xc4" +
|
||||
"\x9e\xb6\x4b\xd2\x6a\xb5\x67\xdf\x92\xde\xdf\xd7\x9c\xab\xd5\xd6" +
|
||||
"\xf4\x89\xfb\xbd\x77\x92\xe4\x58\x6d\x7d\x05\xb4\x5a\x85\x77\x41" +
|
||||
"\xef\xef\xd0\x90\x56\x2b\x32\x1a\xe3\xa8\x24\x55\x11\xa4\xa3\x14" +
|
||||
"\xd9\x9c\x6d\x83\x2b\xbe\xc8\xc3\x05\xd9\xe1\x8b\xc1\x95\xcf\x03" +
|
||||
"\x6d\x01\x8a\xd8\x0e\x9e\xe4\x45\x7a\xa8\x22\x03\x68\x6b\xd3\x77" +
|
||||
"\x4c\x31\x4e\x28\x94\xba\xd3\x19\x90\x02\xa9\x78\x28\x76\x20\xce" +
|
||||
"\x99\x46\x70\x4f\x2b\xdb\x10\x9e\xd4\x22\x9f\xa7\x27\x0c\xee\x50" +
|
||||
"\x78\x1b\xf1\x4f\x84\x5b\xc1\xc6\xc0\x5c\x21\x83\x11\x67\x24\x7b" +
|
||||
"\x22\x86\xb6\x73\x7d\xa0\x8b\xec\x1d\x4c\x6c\x40\x3c\xfd\xa9\xb2" +
|
||||
"\x8e\x4b\xce\x9e\x36\x63\xca\x7b\xba\x18\x0f\x11\xfd\x74\xad\xfb" +
|
||||
"\x10\x5e\x8b\x90\xa0\xc4\x11\x46\x8e\xa0\xef\xe7\xca\x29\x3f\x23" +
|
||||
"\x41\xc3\x41\xd8\x3f\xd6\x54\x59\x0f\xfc\x82\x8b\xf4\x02\xfe\x83" +
|
||||
"\x4b\xcf\x84\x74\x9e\x33\x3b\x35\xf4\x2e\xff\x94\x2e\x21\xbc\xe3" +
|
||||
"\x3f\x4f\x9f\xb0\xd1\x72\x3a\xc2\x8d\xb2\x33\xa3\xa4\x17\xcc\xf4" +
|
||||
"\x8f\x86\x52\x24\x15\xa1\xe2\x15\x53\x7f\x02\x78\x76\x42\x8c\x3c" +
|
||||
"\x43\x6c\xaf\x27\x21\x81\x7b\xd1\xda\xcc\xa6\xf3\xe7\x8e\x69\xe8" +
|
||||
"\xa6\xeb\x40\x73\x0e\xed\x52\xe4\x93\x0d\x0b\xcf\x90\x73\xc3\x7c" +
|
||||
"\xdb\x9f\xb7\xdf\xae\xef\x77\xb2\x2e\x2e\x45\x3e\x59\xa7\x3f\x11" +
|
||||
"\x25\xa1\x0d\x6c\xa4\x0b\x16\x10\xe1\x1a\xa1\xcc\xbf\x99\x36\xb0" +
|
||||
"\xd5\x4d\xba\x82\xb9\x63\xd1\x95\x25\x70\x99\x2c\x67\xeb\x55\xac" +
|
||||
"\x18\x0f\xfe\xa6\xfa\x4f\x00\x00\x00\xff\xff\xdb\x07\xd3\x5c\xad" +
|
||||
"\x1a\x00\x00"
|
||||
|
||||
// DetailedTemplate returns the binary data for a given file.
|
||||
func DetailedTemplate() []byte {
|
||||
// This bit of black magic ensures we do not get
|
||||
// unneccesary memcpy's and can read directly from
|
||||
// the .rodata section.
|
||||
var empty [0]byte
|
||||
sx := (*reflect.StringHeader)(unsafe.Pointer(&_DetailedTemplate))
|
||||
b := empty[:]
|
||||
bx := (*reflect.SliceHeader)(unsafe.Pointer(&b))
|
||||
bx.Data = sx.Data
|
||||
bx.Len = len(_DetailedTemplate)
|
||||
bx.Cap = bx.Len
|
||||
|
||||
gz, err := gzip.NewReader(bytes.NewBuffer(b))
|
||||
|
||||
if err != nil {
|
||||
panic("Decompression failed: " + err.Error())
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
io.Copy(&buf, gz)
|
||||
gz.Close()
|
||||
|
||||
return buf.Bytes()
|
||||
}
|
||||
89
vendor/github.com/JalfResi/justext/htmlRenderer.go
сгенерированный
поставляемый
Обычный файл
89
vendor/github.com/JalfResi/justext/htmlRenderer.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,89 @@
|
||||
package justext
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
|
||||
"github.com/levigross/exp-html"
|
||||
)
|
||||
|
||||
/**
|
||||
This should be a separate package!
|
||||
And it should be a Writer!
|
||||
*/
|
||||
|
||||
var selfClosingTags = map[string]bool{
|
||||
"area": true,
|
||||
"base": true,
|
||||
"basefont": true,
|
||||
"br": true,
|
||||
"hr": true,
|
||||
"input": true,
|
||||
"img": true,
|
||||
"link": true,
|
||||
"meta": true,
|
||||
}
|
||||
|
||||
// nodesToString loops over a node tree and generate HTML string
|
||||
// Should be moved into html/utils package
|
||||
func nodesToString(node *html.Node) string {
|
||||
var response string = ""
|
||||
|
||||
switch node.Type {
|
||||
case html.TextNode:
|
||||
response = html.EscapeString(strings.TrimSpace(node.Data))
|
||||
|
||||
case html.ElementNode, html.DoctypeNode:
|
||||
var att string = ""
|
||||
if len(node.Attr) > 0 {
|
||||
for _, a := range node.Attr {
|
||||
att = fmt.Sprintf("%s %s=\"%s\"", att, a.Key, a.Val)
|
||||
}
|
||||
}
|
||||
|
||||
if _, ok := selfClosingTags[node.Data]; ok {
|
||||
return fmt.Sprintf("<%s%s>", node.Data, att)
|
||||
}
|
||||
|
||||
var content string = ""
|
||||
if len(node.Child) > 0 {
|
||||
for _, n := range node.Child {
|
||||
content = fmt.Sprintf("%s%s", content, nodesToString(n))
|
||||
}
|
||||
}
|
||||
response = fmt.Sprintf("<%s%s>%s</%s>", node.Data, att, content, node.Data)
|
||||
|
||||
case html.DocumentNode:
|
||||
if len(node.Child) > 0 {
|
||||
for _, n := range node.Child {
|
||||
response = nodesToString(n)
|
||||
}
|
||||
}
|
||||
|
||||
case html.CommentNode:
|
||||
// ignore
|
||||
|
||||
default:
|
||||
log.Printf("Unhandled node: %s", nodeTypeToString(node))
|
||||
}
|
||||
return response
|
||||
}
|
||||
|
||||
func nodeTypeToString(n *html.Node) (t string) {
|
||||
switch n.Type {
|
||||
case html.ErrorNode:
|
||||
t = "Error"
|
||||
case html.TextNode:
|
||||
t = "Text"
|
||||
case html.DocumentNode:
|
||||
t = "Document"
|
||||
case html.ElementNode:
|
||||
t = "Element"
|
||||
case html.CommentNode:
|
||||
t = "Comment"
|
||||
case html.DoctypeNode:
|
||||
t = "Doctype"
|
||||
}
|
||||
return t
|
||||
}
|
||||
160
vendor/github.com/JalfResi/justext/paragraphObjectModel.go
сгенерированный
поставляемый
Обычный файл
160
vendor/github.com/JalfResi/justext/paragraphObjectModel.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,160 @@
|
||||
package justext
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/levigross/exp-html"
|
||||
"io"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var (
|
||||
paragraphTags = map[string]bool{
|
||||
"blockquote": true,
|
||||
"caption": true,
|
||||
"center": true,
|
||||
"col": true,
|
||||
"colgroup": true,
|
||||
"dd": true,
|
||||
"div": true,
|
||||
"dl": true,
|
||||
"dt": true,
|
||||
"fieldset": true,
|
||||
"form": true,
|
||||
"legend": true,
|
||||
"optgroup": true,
|
||||
"option": true,
|
||||
"p": true,
|
||||
"pre": true,
|
||||
"table": true,
|
||||
"td": true,
|
||||
"textarea": true,
|
||||
"tfoot": true,
|
||||
"th": true,
|
||||
"thead": true,
|
||||
"tr": true,
|
||||
"ul": true,
|
||||
"li": true,
|
||||
"h1": true,
|
||||
"h2": true,
|
||||
"h3": true,
|
||||
"h4": true,
|
||||
"h5": true,
|
||||
"h6": true,
|
||||
}
|
||||
matchWhiteSpace *regexp.Regexp = regexp.MustCompile("[\n\r\t]+")
|
||||
)
|
||||
|
||||
type Paragraph struct {
|
||||
DomPath string
|
||||
TextNodes []string
|
||||
WordCount int
|
||||
LinkedCharCount int
|
||||
TagCount int
|
||||
Text string
|
||||
StopwordCount int
|
||||
StopwordDensity float64
|
||||
LinkDensity float64
|
||||
Heading bool
|
||||
CfClass string
|
||||
Class string
|
||||
}
|
||||
|
||||
func paragraphObjectModel(htmlStr string) ([]*Paragraph, error) {
|
||||
|
||||
var dom []string
|
||||
var paragraphs []*Paragraph
|
||||
var paragraph *Paragraph = &Paragraph{WordCount: 0, LinkedCharCount: 0, TagCount: 0}
|
||||
var link bool = false
|
||||
var br bool = false
|
||||
var matchToDoErrors *regexp.Regexp = regexp.MustCompile("^html: TODO: ")
|
||||
|
||||
var startNewParagraph func()
|
||||
startNewParagraph = func() {
|
||||
if len(paragraph.TextNodes) != 0 {
|
||||
paragraph.Text = strings.TrimSpace(matchWhiteSpace.ReplaceAllString(strings.Join(paragraph.TextNodes, " "), " "))
|
||||
paragraphs = append(paragraphs, paragraph)
|
||||
}
|
||||
paragraph = &Paragraph{
|
||||
DomPath: strings.Join(dom, "."),
|
||||
WordCount: 0,
|
||||
LinkedCharCount: 0,
|
||||
TagCount: 0,
|
||||
}
|
||||
}
|
||||
|
||||
z := html.NewTokenizer(strings.NewReader(htmlStr))
|
||||
|
||||
for {
|
||||
tt := z.Next()
|
||||
switch tt {
|
||||
|
||||
case html.ErrorToken:
|
||||
if z.Err() == io.EOF {
|
||||
return paragraphs, nil
|
||||
}
|
||||
if matchToDoErrors.MatchString(fmt.Sprintf("%s", z.Err())) {
|
||||
return nil, z.Err()
|
||||
}
|
||||
continue
|
||||
|
||||
case html.StartTagToken:
|
||||
tmpName, _ := z.TagName()
|
||||
name := string(tmpName)
|
||||
//log.Println("Matched start tag: ", name)
|
||||
dom = append(dom, name)
|
||||
_, ok := paragraphTags[name]
|
||||
if ok || (name == "br" && br) {
|
||||
if name == "br" {
|
||||
paragraph.TagCount--
|
||||
}
|
||||
startNewParagraph()
|
||||
} else {
|
||||
if name == "br" {
|
||||
br = true
|
||||
} else {
|
||||
br = false
|
||||
}
|
||||
if name == "a" {
|
||||
link = true
|
||||
}
|
||||
paragraph.TagCount++
|
||||
}
|
||||
|
||||
case html.EndTagToken:
|
||||
tmpName, _ := z.TagName()
|
||||
name := string(tmpName)
|
||||
//log.Println("Matched end tag: ", name)
|
||||
dom = dom[0 : len(dom)-1]
|
||||
if _, ok := paragraphTags[name]; ok {
|
||||
startNewParagraph()
|
||||
}
|
||||
if name == "a" {
|
||||
link = false
|
||||
}
|
||||
|
||||
case html.TextToken:
|
||||
text := strings.TrimSpace(string(z.Text()))
|
||||
e := 15
|
||||
if len(text) < e {
|
||||
e = len(text)
|
||||
}
|
||||
//log.Println("Matched text: ", text[:e], "...")
|
||||
if text == "" {
|
||||
continue
|
||||
}
|
||||
text = strings.TrimSpace(matchWhiteSpace.ReplaceAllString(text, " "))
|
||||
paragraph.TextNodes = append(paragraph.TextNodes, text)
|
||||
words := strings.Split(text, " ")
|
||||
paragraph.WordCount += len(words)
|
||||
if link {
|
||||
paragraph.LinkedCharCount += len(text)
|
||||
}
|
||||
br = false
|
||||
|
||||
}
|
||||
}
|
||||
startNewParagraph()
|
||||
|
||||
return paragraphs, nil
|
||||
}
|
||||
111
vendor/github.com/JalfResi/justext/preprocess.go
сгенерированный
поставляемый
Обычный файл
111
vendor/github.com/JalfResi/justext/preprocess.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,111 @@
|
||||
package justext
|
||||
|
||||
import (
|
||||
"github.com/levigross/exp-html"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func preprocess(htmlStr, encoding, defaultEncoding, encErrors string) (*html.Node, error) {
|
||||
|
||||
root, err := html.Parse(strings.NewReader(htmlStr))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
addKwTags(root)
|
||||
removeElements(root, []string{"head", "script", "style"})
|
||||
|
||||
return root, nil
|
||||
}
|
||||
|
||||
type nodeIterator func(n *html.Node)
|
||||
|
||||
func nodeIter(n *html.Node, f nodeIterator) {
|
||||
f(n)
|
||||
for _, c := range n.Child {
|
||||
nodeIter(c, f)
|
||||
}
|
||||
}
|
||||
|
||||
func addKwTags(root *html.Node) *html.Node {
|
||||
var blankText *regexp.Regexp = regexp.MustCompile("^[\n\r\t ]*$")
|
||||
var nodesWithText []*html.Node
|
||||
|
||||
var markTextAndTail nodeIterator
|
||||
markTextAndTail = func(node *html.Node) {
|
||||
if node.Type != html.CommentNode || node.Type != html.DoctypeNode {
|
||||
if node.Type == html.TextNode {
|
||||
nodesWithText = append(nodesWithText, node)
|
||||
}
|
||||
}
|
||||
}
|
||||
nodeIter(root, markTextAndTail)
|
||||
|
||||
for _, node := range nodesWithText {
|
||||
if blankText.MatchString(node.Data) {
|
||||
node.Data = ""
|
||||
} else {
|
||||
kw := &html.Node{
|
||||
Parent: nil,
|
||||
Type: html.ElementNode,
|
||||
Data: "kw",
|
||||
}
|
||||
node2 := CopyNode(node, true)
|
||||
kw.Child = append(kw.Child, node2)
|
||||
insertNode(node, kw)
|
||||
node.Parent.Remove(node)
|
||||
}
|
||||
}
|
||||
|
||||
return root
|
||||
}
|
||||
|
||||
func removeElements(root *html.Node, elementsToRemove []string) {
|
||||
var toBeRemoved []*html.Node
|
||||
var markRemovableNodes = func(node *html.Node) {
|
||||
if node.Type == html.ElementNode {
|
||||
for _, nodeName := range elementsToRemove {
|
||||
if node.Data == nodeName {
|
||||
toBeRemoved = append(toBeRemoved, node)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
nodeIter(root, markRemovableNodes)
|
||||
|
||||
for _, node := range toBeRemoved {
|
||||
node.Parent.Remove(node)
|
||||
}
|
||||
}
|
||||
|
||||
// insertsNode inserts a Node in a Node tree at the position of another node.
|
||||
// Should be moved into html/utils package
|
||||
func insertNode(originalNode *html.Node, newNode *html.Node) {
|
||||
slice := originalNode.Parent.Child
|
||||
for position, n := range slice {
|
||||
if n == originalNode {
|
||||
originalNode.Parent.Child = append(slice[:position], append([]*html.Node{newNode}, slice[position:]...)...)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func CopyNode(node *html.Node, deep bool) *html.Node {
|
||||
newNode := &html.Node{
|
||||
Type: node.Type,
|
||||
Data: node.Data,
|
||||
}
|
||||
|
||||
if deep && len(node.Child) > 0 {
|
||||
for _, n := range node.Child {
|
||||
newNode.Child = append(newNode.Child, CopyNode(n, true))
|
||||
}
|
||||
}
|
||||
|
||||
for _, i := range node.Attr {
|
||||
newNode.Attr = append(newNode.Attr, html.Attribute{Key: i.Key, Val: i.Val})
|
||||
}
|
||||
|
||||
return newNode
|
||||
}
|
||||
98
vendor/github.com/JalfResi/justext/reader.go
сгенерированный
поставляемый
Обычный файл
98
vendor/github.com/JalfResi/justext/reader.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,98 @@
|
||||
package justext
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/levigross/exp-html"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Reader struct {
|
||||
LengthLow int
|
||||
LengthHigh int
|
||||
Stoplist map[string]bool
|
||||
StopwordsLow float64
|
||||
StopwordsHigh float64
|
||||
MaxLinkDensity float64
|
||||
MaxHeadingDistance int
|
||||
NoHeadings bool
|
||||
r io.Reader
|
||||
}
|
||||
|
||||
func NewReader(r io.Reader) *Reader {
|
||||
return &Reader{
|
||||
LengthLow: 70,
|
||||
LengthHigh: 200,
|
||||
StopwordsLow: 0.30,
|
||||
StopwordsHigh: 0.32,
|
||||
MaxLinkDensity: 0.2,
|
||||
MaxHeadingDistance: 200,
|
||||
NoHeadings: false,
|
||||
r: r,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Reader) ReadAll() ([]*Paragraph, error) {
|
||||
in, err := ioutil.ReadAll(r.r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
root, err := preprocess(string(in), "utf-8", "utf-8", "errors")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if root == nil {
|
||||
return nil, errors.New("Preprocess has resulted in nil")
|
||||
}
|
||||
|
||||
htmlSource := nodesToString(root)
|
||||
if len(htmlSource) == 0 {
|
||||
return nil, errors.New("MAIN: perprocess has returned an empty string")
|
||||
}
|
||||
|
||||
p, err := paragraphObjectModel(htmlSource)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if p == nil {
|
||||
return nil, errors.New("MAIN: P is nil")
|
||||
}
|
||||
|
||||
classifyParagraphs(p, r.Stoplist, r.LengthLow, r.LengthHigh, r.StopwordsLow, r.StopwordsHigh, r.MaxLinkDensity, r.NoHeadings)
|
||||
reviseParagraphClassification(p, r.MaxHeadingDistance)
|
||||
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func dumpNodes(n *html.Node, tab int, exploreChildNodes bool) string {
|
||||
var childNodes string = ""
|
||||
if exploreChildNodes == true {
|
||||
if len(n.Child) > 0 {
|
||||
for _, c := range n.Child {
|
||||
childNodes = fmt.Sprintf("%s%s\n", childNodes, dumpNodes(c, tab+1, true))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var t string
|
||||
switch n.Type {
|
||||
case html.ErrorNode:
|
||||
t = "Err"
|
||||
case html.TextNode:
|
||||
t = "T"
|
||||
case html.DocumentNode:
|
||||
t = "D"
|
||||
case html.ElementNode:
|
||||
t = "E"
|
||||
case html.CommentNode:
|
||||
t = "C"
|
||||
case html.DoctypeNode:
|
||||
t = "Dt"
|
||||
}
|
||||
|
||||
tabStr := strings.Repeat(" ", tab)
|
||||
return fmt.Sprintf("%s%s:%s\n%s", tabStr, t, strings.TrimSpace(strings.Replace(n.Data, "\n", "", -1)), childNodes)
|
||||
}
|
||||
118
vendor/github.com/JalfResi/justext/reviseParagraphClassification.go
сгенерированный
поставляемый
Обычный файл
118
vendor/github.com/JalfResi/justext/reviseParagraphClassification.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,118 @@
|
||||
package justext
|
||||
|
||||
// Context-sensitive paragraph classification. Assumes that classify_pragraphs has already been called.
|
||||
func reviseParagraphClassification(paragraphs []*Paragraph, maxHeadingDistance int) {
|
||||
|
||||
// Copy classes
|
||||
for _, paragraph := range paragraphs {
|
||||
paragraph.Class = paragraph.CfClass
|
||||
}
|
||||
|
||||
// Good headings
|
||||
var j int = 0
|
||||
var distance int
|
||||
for i, paragraph := range paragraphs {
|
||||
if !(paragraph.Heading && paragraph.Class == "short") {
|
||||
continue
|
||||
}
|
||||
|
||||
j = i + 1
|
||||
distance = 0
|
||||
|
||||
for j < len(paragraphs) && distance <= maxHeadingDistance {
|
||||
if paragraphs[j].Class == "good" {
|
||||
paragraph.Class = "neargood"
|
||||
break
|
||||
}
|
||||
distance += len(paragraphs[j].Text)
|
||||
j += 1
|
||||
}
|
||||
}
|
||||
|
||||
// Classify short
|
||||
var newClasses []string = make([]string, len(paragraphs))
|
||||
for i, paragraph := range paragraphs {
|
||||
if paragraph.Class != "short" {
|
||||
continue
|
||||
}
|
||||
|
||||
var prevNeighbour string = getPrevNeighbour(i, paragraphs, true)
|
||||
var nextNeighbour string = getNextNeighbour(i, paragraphs, true)
|
||||
|
||||
var neighbours map[string]bool = make(map[string]bool)
|
||||
neighbours[prevNeighbour] = true
|
||||
neighbours[nextNeighbour] = true
|
||||
|
||||
if _, ok := neighbours["good"]; ok && len(neighbours) == 1 {
|
||||
newClasses[i] = "good"
|
||||
} else if _, ok := neighbours["bad"]; ok && len(neighbours) == 1 {
|
||||
newClasses[i] = "bad"
|
||||
// neighbours must contain both good and bad
|
||||
} else if (prevNeighbour == "bad" && getPrevNeighbour(i, paragraphs, false) == "neargood") || (nextNeighbour == "bad" && getNextNeighbour(i, paragraphs, false) == "neargood") {
|
||||
newClasses[i] = "good"
|
||||
} else {
|
||||
newClasses[i] = "bad"
|
||||
}
|
||||
}
|
||||
|
||||
for i, c := range newClasses {
|
||||
if c != "" {
|
||||
paragraphs[i].Class = c
|
||||
}
|
||||
}
|
||||
|
||||
// revise neargood
|
||||
for i, paragraph := range paragraphs {
|
||||
if paragraph.Class != "neargood" {
|
||||
continue
|
||||
}
|
||||
|
||||
var prevNeighbour string = getPrevNeighbour(i, paragraphs, true)
|
||||
var nextNeighbour string = getNextNeighbour(i, paragraphs, true)
|
||||
|
||||
if prevNeighbour == "bad" && nextNeighbour == "bad" {
|
||||
paragraph.Class = "bad"
|
||||
} else {
|
||||
paragraph.Class = "good"
|
||||
}
|
||||
}
|
||||
|
||||
// more good headings
|
||||
for i, paragraph := range paragraphs {
|
||||
if !(paragraph.Heading && paragraph.Class == "bad" && paragraph.CfClass != "bad") {
|
||||
continue
|
||||
}
|
||||
j = i + 1
|
||||
distance = 0
|
||||
for j < len(paragraphs) && distance <= maxHeadingDistance {
|
||||
if paragraphs[j].Class == "good" {
|
||||
paragraph.Class = "good"
|
||||
break
|
||||
}
|
||||
distance += len(paragraphs[j].Text)
|
||||
j += 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func getPrevNeighbour(i int, paragraphs []*Paragraph, ignoreNeargood bool) string {
|
||||
return getNeighbour(i, paragraphs, ignoreNeargood, -1, -1)
|
||||
}
|
||||
|
||||
func getNextNeighbour(i int, paragraphs []*Paragraph, ignoreNeargood bool) string {
|
||||
return getNeighbour(i, paragraphs, ignoreNeargood, 1, len(paragraphs))
|
||||
}
|
||||
|
||||
func getNeighbour(i int, paragraphs []*Paragraph, ignoreNeargood bool, inc int, boundary int) string {
|
||||
for i+inc != boundary {
|
||||
i += inc
|
||||
var c string = paragraphs[i].Class
|
||||
if c == "good" || c == "bad" {
|
||||
return c
|
||||
}
|
||||
if c == "neargood" && !ignoreNeargood {
|
||||
return c
|
||||
}
|
||||
}
|
||||
return "bad"
|
||||
}
|
||||
158
vendor/github.com/JalfResi/justext/stoplists.go
сгенерированный
поставляемый
Обычный файл
158
vendor/github.com/JalfResi/justext/stoplists.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,158 @@
|
||||
package justext
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
)
|
||||
|
||||
type ResourceFunc func() ([]byte, error)
|
||||
|
||||
var stoplists = map[string]ResourceFunc{}
|
||||
|
||||
func RegisterStoplist(name string, resourceFunc ResourceFunc) {
|
||||
stoplists[name] = resourceFunc
|
||||
}
|
||||
|
||||
/*
|
||||
{
|
||||
"Afrikaans": AfrikaansStoplist,
|
||||
"Albanian": AlbanianStoplist,
|
||||
"Arabic": ArabicStoplist,
|
||||
"Aragonese": AragoneseStoplist,
|
||||
"Armenian": ArmenianStoplist,
|
||||
"Aromanian": AromanianStoplist,
|
||||
"Asturian": AsturianStoplist,
|
||||
"Azerbaijani": AzerbaijaniStoplist,
|
||||
"Basque": BasqueStoplist,
|
||||
"Belarusian": BelarusianStoplist,
|
||||
"Belarusian_Taraskievica": Belarusian_TaraskievicaStoplist,
|
||||
"Bengali": BengaliStoplist,
|
||||
"Bishnupriya_Manipuri": Bishnupriya_ManipuriStoplist,
|
||||
"Bosnian": BosnianStoplist,
|
||||
"Breton": BretonStoplist,
|
||||
"Bulgarian": BulgarianStoplist,
|
||||
"Catalan": CatalanStoplist,
|
||||
"Cebuano": CebuanoStoplist,
|
||||
"Chuvash": ChuvashStoplist,
|
||||
"Croatian": CroatianStoplist,
|
||||
"Czech": CzechStoplist,
|
||||
"Danish": DanishStoplist,
|
||||
"Dutch": DutchStoplist,
|
||||
"English": EnglishStoplist,
|
||||
"Esperanto": EsperantoStoplist,
|
||||
"Estonian": EstonianStoplist,
|
||||
"Finnish": FinnishStoplist,
|
||||
"French": FrenchStoplist,
|
||||
"Galician": GalicianStoplist,
|
||||
"Georgian": GeorgianStoplist,
|
||||
"German": GermanStoplist,
|
||||
"Greek": GreekStoplist,
|
||||
"Gujarati": GujaratiStoplist,
|
||||
"Haitian": HaitianStoplist,
|
||||
"Hebrew": HebrewStoplist,
|
||||
"Hindi": HindiStoplist,
|
||||
"Hungarian": HungarianStoplist,
|
||||
"Icelandic": IcelandicStoplist,
|
||||
"Ido": IdoStoplist,
|
||||
"Igbo": IgboStoplist,
|
||||
"Indonesian": IndonesianStoplist,
|
||||
"Irish": IrishStoplist,
|
||||
"Italian": ItalianStoplist,
|
||||
"Javanese": JavaneseStoplist,
|
||||
"Kannada": KannadaStoplist,
|
||||
"Kazakh": KazakhStoplist,
|
||||
"Korean": KoreanStoplist,
|
||||
"Kurdish": KurdishStoplist,
|
||||
"Kyrgyz": KyrgyzStoplist,
|
||||
"Latin": LatinStoplist,
|
||||
"Latvian": LatvianStoplist,
|
||||
"Lithuanian": LithuanianStoplist,
|
||||
"Lombard": LombardStoplist,
|
||||
"Low_Saxon": Low_SaxonStoplist,
|
||||
"Luxembourgish": LuxembourgishStoplist,
|
||||
"Macedonian": MacedonianStoplist,
|
||||
"Malay": MalayStoplist,
|
||||
"Malayalam": MalayalamStoplist,
|
||||
"Maltese": MalteseStoplist,
|
||||
"Marathi": MarathiStoplist,
|
||||
"Neapolitan": NeapolitanStoplist,
|
||||
"Nepali": NepaliStoplist,
|
||||
"Newar": NewarStoplist,
|
||||
"Norwegian_Bokmal": Norwegian_BokmalStoplist,
|
||||
"Norwegian_Nynorsk": Norwegian_NynorskStoplist,
|
||||
"Occitan": OccitanStoplist,
|
||||
"Persian": PersianStoplist,
|
||||
"Piedmontese": PiedmonteseStoplist,
|
||||
"Polish": PolishStoplist,
|
||||
"Portuguese": PortugueseStoplist,
|
||||
"Quechua": QuechuaStoplist,
|
||||
"Romanian": RomanianStoplist,
|
||||
"Russian": RussianStoplist,
|
||||
"Samogitian": SamogitianStoplist,
|
||||
"Serbian": SerbianStoplist,
|
||||
"Serbo_Croatian": Serbo_CroatianStoplist,
|
||||
"Sicilian": SicilianStoplist,
|
||||
"Simple_English": Simple_EnglishStoplist,
|
||||
"Slovak": SlovakStoplist,
|
||||
"Slovenian": SlovenianStoplist,
|
||||
"Spanish": SpanishStoplist,
|
||||
"Sundanese": SundaneseStoplist,
|
||||
"Swahili": SwahiliStoplist,
|
||||
"Swedish": SwedishStoplist,
|
||||
"Tagalog": TagalogStoplist,
|
||||
"Tamil": TamilStoplist,
|
||||
"Telugu": TeluguStoplist,
|
||||
"Turkish": TurkishStoplist,
|
||||
"Turkmen": TurkmenStoplist,
|
||||
"Ukrainian": UkrainianStoplist,
|
||||
"Urdu": UrduStoplist,
|
||||
"Uzbek": UzbekStoplist,
|
||||
"Vietnamese": VietnameseStoplist,
|
||||
"Volapuk": VolapukStoplist,
|
||||
"Walloon": WalloonStoplist,
|
||||
"Waray_Waray": Waray_WarayStoplist,
|
||||
"Welsh": WelshStoplist,
|
||||
"West_Frisian": West_FrisianStoplist,
|
||||
"Western_Panjabi": Western_PanjabiStoplist,
|
||||
"Yoruba": YorubaStoplist,
|
||||
}
|
||||
*/
|
||||
|
||||
func ReadStoplist(filename string) (map[string]bool, error) {
|
||||
data, err := ioutil.ReadFile(filename)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
db := bytes.Split(data, []uint8("\n"))
|
||||
|
||||
// Convert to map
|
||||
var list = make(map[string]bool)
|
||||
for _, val := range db {
|
||||
list[string(val)] = true
|
||||
}
|
||||
|
||||
return list, nil
|
||||
}
|
||||
|
||||
func GetStoplist(language string) (map[string]bool, error) {
|
||||
if _, ok := stoplists[language]; !ok {
|
||||
return nil, errors.New(fmt.Sprintf("Language %s not supported", language))
|
||||
}
|
||||
|
||||
data, err := stoplists[language]()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
db := bytes.Split(data, []uint8("\n"))
|
||||
|
||||
// Convert to map
|
||||
var list = make(map[string]bool)
|
||||
for _, val := range db {
|
||||
list[string(val)] = true
|
||||
}
|
||||
|
||||
return list, nil
|
||||
}
|
||||
161
vendor/github.com/JalfResi/justext/writer.go
сгенерированный
поставляемый
Обычный файл
161
vendor/github.com/JalfResi/justext/writer.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,161 @@
|
||||
package justext
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"strings"
|
||||
"text/template"
|
||||
)
|
||||
|
||||
// NOTE:
|
||||
// Make a new type:
|
||||
// type JusText []paragraphs
|
||||
|
||||
const (
|
||||
MODE_DEFAULT = 1
|
||||
MODE_DETAILED = 2
|
||||
)
|
||||
|
||||
type Writer struct {
|
||||
Mode int
|
||||
NoBoilerplate bool
|
||||
Stoplist map[string]bool
|
||||
w io.Writer
|
||||
}
|
||||
|
||||
func NewWriter(w io.Writer) *Writer {
|
||||
return &Writer{
|
||||
Mode: MODE_DEFAULT,
|
||||
NoBoilerplate: true,
|
||||
w: w,
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Writer) WriteAll(paragraphs []*Paragraph) error {
|
||||
switch w.Mode {
|
||||
case MODE_DEFAULT:
|
||||
return w.outputDefault(paragraphs)
|
||||
break
|
||||
|
||||
case MODE_DETAILED:
|
||||
return w.outputDetailed(paragraphs)
|
||||
break
|
||||
|
||||
default:
|
||||
return errors.New("Unrecognised mode")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func IsGood(args ...interface{}) (result bool) {
|
||||
result = true
|
||||
for _, val := range args {
|
||||
if val != "good" {
|
||||
result = false
|
||||
return
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (w *Writer) outputDefault(paragraphs []*Paragraph) error {
|
||||
templateData := DefaultTemplate()
|
||||
t := template.New("default")
|
||||
t.Funcs(template.FuncMap{"TrimSpace": strings.TrimSpace})
|
||||
t.Funcs(template.FuncMap{"IsGood": IsGood})
|
||||
|
||||
templ, err := t.Parse(string(templateData))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var data = struct {
|
||||
Paragraphs []*Paragraph
|
||||
NoBoilerplate bool
|
||||
}{paragraphs, w.NoBoilerplate}
|
||||
|
||||
return templ.Execute(w.w, data)
|
||||
}
|
||||
|
||||
func (w *Writer) outputDetailed(paragraphs []*Paragraph) error {
|
||||
templateData := DetailedTemplate()
|
||||
var markStopwords func(args ...interface{}) string
|
||||
markStopwords = func(args ...interface{}) string {
|
||||
|
||||
var output string = ""
|
||||
words := strings.Split(args[0].(string), " ")
|
||||
for _, word := range words {
|
||||
if _, ok := w.Stoplist[strings.TrimSpace(word)]; ok {
|
||||
output = fmt.Sprintf("%s<span class=\"stopword\">%s</span> ", output, word)
|
||||
} else {
|
||||
output = fmt.Sprintf("%s%s ", output, word)
|
||||
}
|
||||
}
|
||||
|
||||
return output
|
||||
}
|
||||
|
||||
t := template.New("detailed")
|
||||
t.Funcs(template.FuncMap{"TrimSpace": strings.TrimSpace})
|
||||
t.Funcs(template.FuncMap{"MarkStopwords": markStopwords})
|
||||
|
||||
templ, err := t.Parse(string(templateData))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var data = struct {
|
||||
Paragraphs []*Paragraph
|
||||
}{paragraphs}
|
||||
|
||||
return templ.Execute(w.w, data)
|
||||
}
|
||||
|
||||
func (w *Writer) OutputDebug(paragraphs []*Paragraph) {
|
||||
for _, paragraph := range paragraphs {
|
||||
log.Println(paragraph.DomPath)
|
||||
log.Println("\tfinal class: ", paragraph.Class)
|
||||
log.Println("\tcontext-free class: ", paragraph.CfClass)
|
||||
log.Println("\theading: ", paragraph.Heading)
|
||||
log.Println("\tlength (in characters): ", len(paragraph.Text))
|
||||
log.Println("\tnumber of characters with links: ", paragraph.LinkedCharCount)
|
||||
log.Println("\tlink density: ", paragraph.LinkDensity)
|
||||
log.Println("\tnumber of words: ", paragraph.WordCount)
|
||||
log.Println("\tnumber of stop words: ", paragraph.StopwordCount)
|
||||
log.Println("\tstop word density: ", paragraph.StopwordDensity)
|
||||
}
|
||||
}
|
||||
|
||||
// TO-DO:
|
||||
// Need an output feature that returns a de-duped space separated text file of all the
|
||||
// words in the output document sans-boilerplate. Also needs option to exclude stoplist
|
||||
// words from that output too.
|
||||
|
||||
// TO-DO:
|
||||
// Need an output feature that returns the content of a stop list (or do we just make
|
||||
// the function getStoplist public? Might be a lot easier...)
|
||||
|
||||
/*
|
||||
func (w *Writer) outputKrdwrd(paragraphs []*Paragraph) (output string) {
|
||||
for _, paragraph := range paragraphs {
|
||||
var cls int
|
||||
if paragraph.Class == "good" || paragraph.Class == "neargood" {
|
||||
if paragraph.Heading {
|
||||
cls = 2
|
||||
} else {
|
||||
cls = 3
|
||||
}
|
||||
} else {
|
||||
cls = 1
|
||||
}
|
||||
for _, textNode := range paragraph.TextNodes {
|
||||
output = fmt.Sprintf("%s%i\t%s", output, cls, strings.TrimSpace(textNode))
|
||||
}
|
||||
}
|
||||
|
||||
return output
|
||||
}
|
||||
*/
|
||||
Ссылка в новой задаче
Block a user