#4257 Added functionality to create previews for post links using open graph data from those links. (#4890)

Этот коммит содержится в:
Debanshu Kundu
2017-01-20 23:11:13 +05:30
коммит произвёл enahum
родитель fefe4b70d9
Коммит 3aaf71fdea
26 изменённых файлов: 1241 добавлений и 534 удалений

24
vendor/github.com/dyatlov/go-opengraph/.gitignore сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,24 @@
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
*.test
*.prof

22
vendor/github.com/dyatlov/go-opengraph/LICENSE сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,22 @@
The MIT License (MIT)
Copyright (c) 2015 Vitaly Dyatlov
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

118
vendor/github.com/dyatlov/go-opengraph/README.md сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,118 @@
Go OpenGraph
===
Parses given html data into Facebook OpenGraph structure.
To download and install this package run:
`go get github.com/dyatlov/go-opengraph/opengraph`
Methods:
* `NewOpenGraph()` - create a new OpenGraph instance
* `ProcessHTML(buffer io.Reader) error` - process given html into underlying data structure
* `ProcessMeta(metaAttrs map[string]string)` - add data to the structure based on meta attributes
* `ToJSON() (string, error)` - return JSON representation of data or error
* `String() string` - return JSON representation of structure
Source docs: http://godoc.org/github.com/dyatlov/go-opengraph/opengraph
If you just need to parse an OpenGraph data from HTML then method `ProcessHTML` is your needed one.
Example:
```go
package main
import (
"fmt"
"strings"
"github.com/dyatlov/go-opengraph/opengraph"
)
func main() {
html := `<html><head><meta property="og:type" content="article" />
<meta property="og:title" content="WordPress 4.3 &quot;Billie&quot;" />
<meta property="og:url" content="https://wordpress.org/news/2015/08/billie/" /></head><body></body></html>`
og := opengraph.NewOpenGraph()
err := og.ProcessHTML(strings.NewReader(html))
if err != nil {
fmt.Println(err)
return
}
fmt.Printf("Type: %s\n", og.Type)
fmt.Printf("Title: %s\n", og.Title)
fmt.Printf("URL: %s\n", og.URL)
fmt.Printf("String/JSON Representation: %s\n", og)
}
```
If you have your own parsing engine and just need an intelligent OpenGraph parsing, then `ProcessMeta` is the method you need.
While using this method you don't need to reparse your parsed html again, just feed it with meta atributes as they appear and OpenGraph will be built based on the data.
Example:
```go
package main
import (
"fmt"
"strings"
"github.com/dyatlov/go-opengraph/opengraph"
"golang.org/x/net/html"
)
func main() {
h := `<html><head><meta property="og:type" content="article" />
<meta property="og:title" content="WordPress 4.3 &quot;Billie&quot;" />
<meta property="og:url" content="https://wordpress.org/news/2015/08/billie/" /></head><body></body></html>`
og := opengraph.NewOpenGraph()
doc, err := html.Parse(strings.NewReader(h))
if err != nil {
fmt.Println(err)
return
}
var parseHead func(*html.Node)
parseHead = func(n *html.Node) {
for c := n.FirstChild; c != nil; c = c.NextSibling {
if c.Type == html.ElementNode && c.Data == "meta" {
m := make(map[string]string)
for _, a := range c.Attr {
m[a.Key] = a.Val
}
og.ProcessMeta(m)
}
}
}
var f func(*html.Node)
f = func(n *html.Node) {
for c := n.FirstChild; c != nil; c = c.NextSibling {
if c.Type == html.ElementNode {
if c.Data == "head" {
parseHead(c)
continue
} else if c.Data == "body" { // OpenGraph is only in head, so we don't need body
break
}
}
f(c)
}
}
f(doc)
fmt.Printf("Type: %s\n", og.Type)
fmt.Printf("Title: %s\n", og.Title)
fmt.Printf("URL: %s\n", og.URL)
fmt.Printf("String/JSON Representation: %s\n", og)
}
```

58
vendor/github.com/dyatlov/go-opengraph/examples/advanced.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,58 @@
package main
import (
"fmt"
"strings"
"github.com/dyatlov/go-opengraph/opengraph"
"golang.org/x/net/html"
)
func main() {
h := `<html><head><meta property="og:type" content="article" />
<meta property="og:title" content="WordPress 4.3 &quot;Billie&quot;" />
<meta property="og:url" content="https://wordpress.org/news/2015/08/billie/" /></head><body></body></html>`
og := opengraph.NewOpenGraph()
doc, err := html.Parse(strings.NewReader(h))
if err != nil {
fmt.Println(err)
return
}
var parseHead func(*html.Node)
parseHead = func(n *html.Node) {
for c := n.FirstChild; c != nil; c = c.NextSibling {
if c.Type == html.ElementNode && c.Data == "meta" {
m := make(map[string]string)
for _, a := range c.Attr {
m[a.Key] = a.Val
}
og.ProcessMeta(m)
}
}
}
var f func(*html.Node)
f = func(n *html.Node) {
for c := n.FirstChild; c != nil; c = c.NextSibling {
if c.Type == html.ElementNode {
if c.Data == "head" {
parseHead(c)
continue
} else if c.Data == "body" { // OpenGraph is only in head, so we don't need body
break
}
}
f(c)
}
}
f(doc)
fmt.Printf("Type: %s\n", og.Type)
fmt.Printf("Title: %s\n", og.Title)
fmt.Printf("URL: %s\n", og.URL)
fmt.Printf("String/JSON Representation: %s\n", og)
}

27
vendor/github.com/dyatlov/go-opengraph/examples/simple.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,27 @@
package main
import (
"fmt"
"strings"
"github.com/dyatlov/go-opengraph/opengraph"
)
func main() {
html := `<html><head><meta property="og:type" content="article" />
<meta property="og:title" content="WordPress 4.3 &quot;Billie&quot;" />
<meta property="og:url" content="https://wordpress.org/news/2015/08/billie/" /></head><body></body></html>`
og := opengraph.NewOpenGraph()
err := og.ProcessHTML(strings.NewReader(html))
if err != nil {
fmt.Println(err)
return
}
fmt.Printf("Type: %s\n", og.Type)
fmt.Printf("Title: %s\n", og.Title)
fmt.Printf("URL: %s\n", og.URL)
fmt.Printf("String/JSON Representation: %s\n", og)
}

329
vendor/github.com/dyatlov/go-opengraph/opengraph/opengraph.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,329 @@
package opengraph
import (
"encoding/json"
"io"
"strconv"
"time"
"golang.org/x/net/html"
"golang.org/x/net/html/atom"
)
// Image defines Open Graph Image type
type Image struct {
URL string `json:"url"`
SecureURL string `json:"secure_url"`
Type string `json:"type"`
Width uint64 `json:"width"`
Height uint64 `json:"height"`
}
// Video defines Open Graph Video type
type Video struct {
URL string `json:"url"`
SecureURL string `json:"secure_url"`
Type string `json:"type"`
Width uint64 `json:"width"`
Height uint64 `json:"height"`
}
// Audio defines Open Graph Audio Type
type Audio struct {
URL string `json:"url"`
SecureURL string `json:"secure_url"`
Type string `json:"type"`
}
// Article contain Open Graph Article structure
type Article struct {
PublishedTime *time.Time `json:"published_time"`
ModifiedTime *time.Time `json:"modified_time"`
ExpirationTime *time.Time `json:"expiration_time"`
Section string `json:"section"`
Tags []string `json:"tags"`
Authors []*Profile `json:"authors"`
}
// Profile contains Open Graph Profile structure
type Profile struct {
FirstName string `json:"first_name"`
LastName string `json:"last_name"`
Username string `json:"username"`
Gender string `json:"gender"`
}
// Book contains Open Graph Book structure
type Book struct {
ISBN string `json:"isbn"`
ReleaseDate *time.Time `json:"release_date"`
Tags []string `json:"tags"`
Authors []*Profile `json:"authors"`
}
// OpenGraph contains facebook og data
type OpenGraph struct {
isArticle bool
isBook bool
isProfile bool
Type string `json:"type"`
URL string `json:"url"`
Title string `json:"title"`
Description string `json:"description"`
Determiner string `json:"determiner"`
SiteName string `json:"site_name"`
Locale string `json:"locale"`
LocalesAlternate []string `json:"locales_alternate"`
Images []*Image `json:"images"`
Audios []*Audio `json:"audios"`
Videos []*Video `json:"videos"`
Article *Article `json:"article,omitempty"`
Book *Book `json:"book,omitempty"`
Profile *Profile `json:"profile,omitempty"`
}
// NewOpenGraph returns new instance of Open Graph structure
func NewOpenGraph() *OpenGraph {
return &OpenGraph{}
}
// ToJSON a simple wrapper around json.Marshal
func (og *OpenGraph) ToJSON() ([]byte, error) {
return json.Marshal(og)
}
// String return json representation of structure, or error string
func (og *OpenGraph) String() string {
data, err := og.ToJSON()
if err != nil {
return err.Error()
}
return string(data[:])
}
// ProcessHTML parses given html from Reader interface and fills up OpenGraph structure
func (og *OpenGraph) ProcessHTML(buffer io.Reader) error {
z := html.NewTokenizer(buffer)
for {
tt := z.Next()
switch tt {
case html.ErrorToken:
if z.Err() == io.EOF {
return nil
}
return z.Err()
case html.StartTagToken, html.SelfClosingTagToken, html.EndTagToken:
name, hasAttr := z.TagName()
if atom.Lookup(name) == atom.Body {
return nil // OpenGraph is only in head, so we don't need body
}
if atom.Lookup(name) != atom.Meta || !hasAttr {
continue
}
m := make(map[string]string)
var key, val []byte
for hasAttr {
key, val, hasAttr = z.TagAttr()
m[atom.String(key)] = string(val)
}
og.ProcessMeta(m)
}
}
return nil
}
// ProcessMeta processes meta attributes and adds them to Open Graph structure if they are suitable for that
func (og *OpenGraph) ProcessMeta(metaAttrs map[string]string) {
switch metaAttrs["property"] {
case "og:description":
og.Description = metaAttrs["content"]
case "og:type":
og.Type = metaAttrs["content"]
switch og.Type {
case "article":
og.isArticle = true
case "book":
og.isBook = true
case "profile":
og.isProfile = true
}
case "og:title":
og.Title = metaAttrs["content"]
case "og:url":
og.URL = metaAttrs["content"]
case "og:determiner":
og.Determiner = metaAttrs["content"]
case "og:site_name":
og.SiteName = metaAttrs["content"]
case "og:locale":
og.Locale = metaAttrs["content"]
case "og:locale:alternate":
og.LocalesAlternate = append(og.LocalesAlternate, metaAttrs["content"])
case "og:image":
og.Images = append(og.Images, &Image{URL: metaAttrs["content"]})
case "og:image:url":
if len(og.Images) > 0 {
og.Images[len(og.Images)-1].URL = metaAttrs["content"]
}
case "og:image:secure_url":
if len(og.Images) > 0 {
og.Images[len(og.Images)-1].SecureURL = metaAttrs["content"]
}
case "og:image:type":
if len(og.Images) > 0 {
og.Images[len(og.Images)-1].Type = metaAttrs["content"]
}
case "og:image:width":
if len(og.Images) > 0 {
w, err := strconv.ParseUint(metaAttrs["content"], 10, 64)
if err == nil {
og.Images[len(og.Images)-1].Width = w
}
}
case "og:image:height":
if len(og.Images) > 0 {
h, err := strconv.ParseUint(metaAttrs["content"], 10, 64)
if err == nil {
og.Images[len(og.Images)-1].Height = h
}
}
case "og:video":
og.Videos = append(og.Videos, &Video{URL: metaAttrs["content"]})
case "og:video:url":
if len(og.Videos) > 0 {
og.Videos[len(og.Videos)-1].URL = metaAttrs["content"]
}
case "og:video:secure_url":
if len(og.Videos) > 0 {
og.Videos[len(og.Videos)-1].SecureURL = metaAttrs["content"]
}
case "og:video:type":
if len(og.Videos) > 0 {
og.Videos[len(og.Videos)-1].Type = metaAttrs["content"]
}
case "og:video:width":
if len(og.Videos) > 0 {
w, err := strconv.ParseUint(metaAttrs["content"], 10, 64)
if err == nil {
og.Videos[len(og.Videos)-1].Width = w
}
}
case "og:video:height":
if len(og.Videos) > 0 {
h, err := strconv.ParseUint(metaAttrs["content"], 10, 64)
if err == nil {
og.Videos[len(og.Videos)-1].Height = h
}
}
default:
if og.isArticle {
og.processArticleMeta(metaAttrs)
} else if og.isBook {
og.processBookMeta(metaAttrs)
} else if og.isProfile {
og.processProfileMeta(metaAttrs)
}
}
}
func (og *OpenGraph) processArticleMeta(metaAttrs map[string]string) {
if og.Article == nil {
og.Article = &Article{}
}
switch metaAttrs["property"] {
case "article:published_time":
t, err := time.Parse(time.RFC3339, metaAttrs["content"])
if err == nil {
og.Article.PublishedTime = &t
}
case "article:modified_time":
t, err := time.Parse(time.RFC3339, metaAttrs["content"])
if err == nil {
og.Article.ModifiedTime = &t
}
case "article:expiration_time":
t, err := time.Parse(time.RFC3339, metaAttrs["content"])
if err == nil {
og.Article.ExpirationTime = &t
}
case "article:secttion":
og.Article.Section = metaAttrs["content"]
case "article:tag":
og.Article.Tags = append(og.Article.Tags, metaAttrs["content"])
case "article:author:first_name":
if len(og.Article.Authors) == 0 {
og.Article.Authors = append(og.Article.Authors, &Profile{})
}
og.Article.Authors[len(og.Article.Authors)-1].FirstName = metaAttrs["content"]
case "article:author:last_name":
if len(og.Article.Authors) == 0 {
og.Article.Authors = append(og.Article.Authors, &Profile{})
}
og.Article.Authors[len(og.Article.Authors)-1].LastName = metaAttrs["content"]
case "article:author:username":
if len(og.Article.Authors) == 0 {
og.Article.Authors = append(og.Article.Authors, &Profile{})
}
og.Article.Authors[len(og.Article.Authors)-1].Username = metaAttrs["content"]
case "article:author:gender":
if len(og.Article.Authors) == 0 {
og.Article.Authors = append(og.Article.Authors, &Profile{})
}
og.Article.Authors[len(og.Article.Authors)-1].Gender = metaAttrs["content"]
}
}
func (og *OpenGraph) processBookMeta(metaAttrs map[string]string) {
if og.Book == nil {
og.Book = &Book{}
}
switch metaAttrs["property"] {
case "book:release_date":
t, err := time.Parse(time.RFC3339, metaAttrs["content"])
if err == nil {
og.Book.ReleaseDate = &t
}
case "book:isbn":
og.Book.ISBN = metaAttrs["content"]
case "book:tag":
og.Book.Tags = append(og.Book.Tags, metaAttrs["content"])
case "book:author:first_name":
if len(og.Book.Authors) == 0 {
og.Book.Authors = append(og.Book.Authors, &Profile{})
}
og.Book.Authors[len(og.Book.Authors)-1].FirstName = metaAttrs["content"]
case "book:author:last_name":
if len(og.Book.Authors) == 0 {
og.Book.Authors = append(og.Book.Authors, &Profile{})
}
og.Book.Authors[len(og.Book.Authors)-1].LastName = metaAttrs["content"]
case "book:author:username":
if len(og.Book.Authors) == 0 {
og.Book.Authors = append(og.Book.Authors, &Profile{})
}
og.Book.Authors[len(og.Book.Authors)-1].Username = metaAttrs["content"]
case "book:author:gender":
if len(og.Book.Authors) == 0 {
og.Book.Authors = append(og.Book.Authors, &Profile{})
}
og.Book.Authors[len(og.Book.Authors)-1].Gender = metaAttrs["content"]
}
}
func (og *OpenGraph) processProfileMeta(metaAttrs map[string]string) {
if og.Profile == nil {
og.Profile = &Profile{}
}
switch metaAttrs["property"] {
case "profile:first_name":
og.Profile.FirstName = metaAttrs["content"]
case "profile:last_name":
og.Profile.LastName = metaAttrs["content"]
case "profile:username":
og.Profile.Username = metaAttrs["content"]
case "profile:gender":
og.Profile.Gender = metaAttrs["content"]
}
}

131
vendor/github.com/dyatlov/go-opengraph/opengraph/opengraph_test.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,131 @@
package opengraph_test
import (
"strings"
"testing"
"time"
"github.com/dyatlov/go-opengraph/opengraph"
)
const html = `
<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml" dir="ltr" lang="en-US">
<head profile="http://gmpg.org/xfn/11">
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>WordPress &#8250; WordPress 4.3 &#8220;Billie&#8221;</title>
<!-- Jetpack Open Graph Tags -->
<meta property="og:type" content="article" />
<meta property="og:title" content="WordPress 4.3 &quot;Billie&quot;" />
<meta property="og:url" content="https://wordpress.org/news/2015/08/billie/" />
<meta property="og:description" content="Version 4.3 of WordPress, named &quot;Billie&quot; in honor of jazz singer Billie Holiday, is available for download or update in your WordPress dashboard. New features in 4.3 make it even easier to format y..." />
<meta property="article:published_time" content="2015-08-18T19:12:38+00:00" />
<meta property="article:modified_time" content="2015-08-19T13:10:24+00:00" />
<meta property="og:site_name" content="WordPress News" />
<meta property="og:image" content="https://www.gravatar.com/avatar/2370ea5912750f4cb0f3c51ae1cbca55?d=mm&amp;s=180&amp;r=G" />
<meta property="og:locale" content="en_US" />
<meta name="twitter:site" content="@WordPress" />
<meta name="twitter:card" content="summary" />
<meta name="twitter:creator" content="@WordPress" />
`
func BenchmarkOpenGraph_ProcessHTML(b *testing.B) {
og := opengraph.NewOpenGraph()
b.ReportAllocs()
b.SetBytes(int64(len(html)))
for i := 0; i < b.N; i++ {
if err := og.ProcessHTML(strings.NewReader(html)); err != nil {
b.Fatal(err)
}
}
}
func TestOpenGraphProcessHTML(t *testing.T) {
og := opengraph.NewOpenGraph()
err := og.ProcessHTML(strings.NewReader(html))
if err != nil {
t.Fatal(err)
}
if og.Type != "article" {
t.Error("type parsed incorrectly")
}
if len(og.Title) == 0 {
t.Error("title parsed incorrectly")
}
if len(og.URL) == 0 {
t.Error("url parsed incorrectly")
}
if len(og.Description) == 0 {
t.Error("description parsed incorrectly")
}
if len(og.Images) == 0 {
t.Error("images parsed incorrectly")
} else {
if len(og.Images[0].URL) == 0 {
t.Error("image url parsed incorrectly")
}
}
if len(og.Locale) == 0 {
t.Error("locale parsed incorrectly")
}
if len(og.SiteName) == 0 {
t.Error("site name parsed incorrectly")
}
if og.Article == nil {
t.Error("articles parsed incorrectly")
} else {
ev, _ := time.Parse(time.RFC3339, "2015-08-18T19:12:38+00:00")
if !og.Article.PublishedTime.Equal(ev) {
t.Error("article published time parsed incorrectly")
}
}
}
func TestOpenGraphProcessMeta(t *testing.T) {
og := opengraph.NewOpenGraph()
og.ProcessMeta(map[string]string{"property": "og:type", "content": "book"})
if og.Type != "book" {
t.Error("wrong og:type processing")
}
og.ProcessMeta(map[string]string{"property": "book:isbn", "content": "123456"})
if og.Book == nil {
t.Error("wrong book type processing")
} else {
if og.Book.ISBN != "123456" {
t.Error("wrong book isbn processing")
}
}
og.ProcessMeta(map[string]string{"property": "article:section", "content": "testsection"})
if og.Article != nil {
t.Error("article processed when it should not be")
}
og.ProcessMeta(map[string]string{"property": "book:author:first_name", "content": "John"})
if og.Book != nil {
if len(og.Book.Authors) == 0 {
t.Error("book author was not processed")
} else {
if og.Book.Authors[0].FirstName != "John" {
t.Error("author first name was processed incorrectly")
}
}
}
}