mirrors/mostlymatter
1
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

MM-40676: Bump dependencies (#19525)

Просмотр исходного кода 
https://mattermost.atlassian.net/browse/MM-40676

```release-note
NONE
```
Этот коммит содержится в:
Agniva De Sarker
2022-02-09 23:05:16 +05:30
коммит произвёл GitHub
родитель 440790dad6
Коммит 57eafbc6ca
322 изменённых файлов: 24889 добавлений и 14797 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

2
vendor/github.com/Masterminds/squirrel/expr.go сгенерированный поставляемый
Просмотреть файл

@@ -372,7 +372,7 @@ func (c conj) join(sep, defaultExpr string) (sql string, args []interface{}, err
}
var sqlParts []string
for _, sqlizer := range c {
partSQL, partArgs, err := sqlizer.ToSql()
partSQL, partArgs, err := nestedToSql(sqlizer)
if err != nil {
return "", nil, err
}

40
vendor/github.com/aws/aws-sdk-go/aws/credentials/stscreds/web_identity_provider.go сгенерированный поставляемый
Просмотреть файл

@@ -28,7 +28,7 @@ const (
// compare test values.
var now = time.Now
// TokenFetcher shuold return WebIdentity token bytes or an error
// TokenFetcher should return WebIdentity token bytes or an error
type TokenFetcher interface {
FetchToken(credentials.Context) ([]byte, error)
}
@@ -50,6 +50,8 @@ func (f FetchTokenPath) FetchToken(ctx credentials.Context) ([]byte, error) {
// an OIDC token.
type WebIdentityRoleProvider struct {
credentials.Expiry
// The policy ARNs to use with the web identity assumed role.
PolicyArns []*sts.PolicyDescriptorType
// Duration the STS credentials will be valid for. Truncated to seconds.
@@ -74,6 +76,9 @@ type WebIdentityRoleProvider struct {
// NewWebIdentityCredentials will return a new set of credentials with a given
// configuration, role arn, and token file path.
//
// Deprecated: Use NewWebIdentityRoleProviderWithOptions for flexible
// functional options, and wrap with credentials.NewCredentials helper.
func NewWebIdentityCredentials(c client.ConfigProvider, roleARN, roleSessionName, path string) *credentials.Credentials {
svc := sts.New(c)
p := NewWebIdentityRoleProvider(svc, roleARN, roleSessionName, path)
@@ -82,19 +87,42 @@ func NewWebIdentityCredentials(c client.ConfigProvider, roleARN, roleSessionName
// NewWebIdentityRoleProvider will return a new WebIdentityRoleProvider with the
// provided stsiface.STSAPI
//
// Deprecated: Use NewWebIdentityRoleProviderWithOptions for flexible
// functional options.
func NewWebIdentityRoleProvider(svc stsiface.STSAPI, roleARN, roleSessionName, path string) *WebIdentityRoleProvider {
return NewWebIdentityRoleProviderWithToken(svc, roleARN, roleSessionName, FetchTokenPath(path))
return NewWebIdentityRoleProviderWithOptions(svc, roleARN, roleSessionName, FetchTokenPath(path))
}
// NewWebIdentityRoleProviderWithToken will return a new WebIdentityRoleProvider with the
// provided stsiface.STSAPI and a TokenFetcher
//
// Deprecated: Use NewWebIdentityRoleProviderWithOptions for flexible
// functional options.
func NewWebIdentityRoleProviderWithToken(svc stsiface.STSAPI, roleARN, roleSessionName string, tokenFetcher TokenFetcher) *WebIdentityRoleProvider {
return &WebIdentityRoleProvider{
return NewWebIdentityRoleProviderWithOptions(svc, roleARN, roleSessionName, tokenFetcher)
}
// NewWebIdentityRoleProviderWithOptions will return an initialize
// WebIdentityRoleProvider with the provided stsiface.STSAPI, role ARN, and a
// TokenFetcher. Additional options can be provided as functional options.
//
// TokenFetcher is the implementation that will retrieve the JWT token from to
// assume the role with. Use the provided FetchTokenPath implementation to
// retrieve the JWT token using a file system path.
func NewWebIdentityRoleProviderWithOptions(svc stsiface.STSAPI, roleARN, roleSessionName string, tokenFetcher TokenFetcher, optFns ...func(*WebIdentityRoleProvider)) *WebIdentityRoleProvider {
p := WebIdentityRoleProvider{
client: svc,
tokenFetcher: tokenFetcher,
roleARN: roleARN,
roleSessionName: roleSessionName,
}
for _, fn := range optFns {
fn(&p)
}
return &p
}
// Retrieve will attempt to assume a role from a token which is located at
@@ -104,9 +132,9 @@ func (p *WebIdentityRoleProvider) Retrieve() (credentials.Value, error) {
return p.RetrieveWithContext(aws.BackgroundContext())
}
// RetrieveWithContext will attempt to assume a role from a token which is located at
// 'WebIdentityTokenFilePath' specified destination and if that is empty an
// error will be returned.
// RetrieveWithContext will attempt to assume a role from a token which is
// located at 'WebIdentityTokenFilePath' specified destination and if that is
// empty an error will be returned.
func (p *WebIdentityRoleProvider) RetrieveWithContext(ctx credentials.Context) (credentials.Value, error) {
b, err := p.tokenFetcher.FetchToken(ctx)
if err != nil {

1060
vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go сгенерированный поставляемый
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

9
vendor/github.com/aws/aws-sdk-go/aws/request/request.go сгенерированный поставляемый
Просмотреть файл

@@ -4,6 +4,7 @@ import (
"bytes"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"reflect"
@@ -525,6 +526,14 @@ func (r *Request) GetBody() io.ReadSeeker {
// Send will not close the request.Request's body.
func (r *Request) Send() error {
defer func() {
// Ensure a non-nil HTTPResponse parameter is set to ensure handlers
// checking for HTTPResponse values, don't fail.
if r.HTTPResponse == nil {
r.HTTPResponse = &http.Response{
Header: http.Header{},
Body: ioutil.NopCloser(&bytes.Buffer{}),
}
}
// Regardless of success or failure of the request trigger the Complete
// request handlers.
r.Handlers.Complete.Run(r)

33
vendor/github.com/aws/aws-sdk-go/aws/session/credentials.go сгенерированный поставляемый
Просмотреть файл

@@ -14,8 +14,17 @@ import (
"github.com/aws/aws-sdk-go/aws/defaults"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/internal/shareddefaults"
"github.com/aws/aws-sdk-go/service/sts"
)
// CredentialsProviderOptions specifies additional options for configuring
// credentials providers.
type CredentialsProviderOptions struct {
// WebIdentityRoleProviderOptions configures a WebIdentityRoleProvider,
// such as setting its ExpiryWindow.
WebIdentityRoleProviderOptions func(*stscreds.WebIdentityRoleProvider)
}
func resolveCredentials(cfg *aws.Config,
envCfg envConfig, sharedCfg sharedConfig,
handlers request.Handlers,
@@ -40,6 +49,7 @@ func resolveCredentials(cfg *aws.Config,
envCfg.WebIdentityTokenFilePath,
envCfg.RoleARN,
envCfg.RoleSessionName,
sessOpts.CredentialsProviderOptions,
)
default:
@@ -59,6 +69,7 @@ var WebIdentityEmptyTokenFilePathErr = awserr.New(stscreds.ErrCodeWebIdentity, "
func assumeWebIdentity(cfg *aws.Config, handlers request.Handlers,
filepath string,
roleARN, sessionName string,
credOptions *CredentialsProviderOptions,
) (*credentials.Credentials, error) {
if len(filepath) == 0 {
@@ -69,17 +80,18 @@ func assumeWebIdentity(cfg *aws.Config, handlers request.Handlers,
return nil, WebIdentityEmptyRoleARNErr
}
creds := stscreds.NewWebIdentityCredentials(
&Session{
Config: cfg,
Handlers: handlers.Copy(),
},
roleARN,
sessionName,
filepath,
)
svc := sts.New(&Session{
Config: cfg,
Handlers: handlers.Copy(),
})
return creds, nil
var optFns []func(*stscreds.WebIdentityRoleProvider)
if credOptions != nil && credOptions.WebIdentityRoleProviderOptions != nil {
optFns = append(optFns, credOptions.WebIdentityRoleProviderOptions)
}
p := stscreds.NewWebIdentityRoleProviderWithOptions(svc, roleARN, sessionName, stscreds.FetchTokenPath(filepath), optFns...)
return credentials.NewCredentials(p), nil
}
func resolveCredsFromProfile(cfg *aws.Config,
@@ -114,6 +126,7 @@ func resolveCredsFromProfile(cfg *aws.Config,
sharedCfg.WebIdentityTokenFile,
sharedCfg.RoleARN,
sharedCfg.RoleSessionName,
sessOpts.CredentialsProviderOptions,
)
case sharedCfg.hasSSOConfiguration():

5
vendor/github.com/aws/aws-sdk-go/aws/session/session.go сгенерированный поставляемый
Просмотреть файл

@@ -304,6 +304,11 @@ type Options struct {
//
// AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE=IPv6
EC2IMDSEndpointMode endpoints.EC2IMDSEndpointModeState
// Specifies options for creating credential providers.
// These are only used if the aws.Config does not already
// include credentials.
CredentialsProviderOptions *CredentialsProviderOptions
}
// NewSessionWithOptions returns a new Session created from SDK defaults, config files,

2
vendor/github.com/aws/aws-sdk-go/aws/version.go сгенерированный поставляемый
Просмотреть файл

@@ -5,4 +5,4 @@ package aws
const SDKName = "aws-sdk-go"
// SDKVersion is the version of this SDK
const SDKVersion = "1.42.11"
const SDKVersion = "1.42.49"

2
vendor/github.com/aws/aws-sdk-go/private/protocol/rest/unmarshal.go сгенерированный поставляемый
Просмотреть файл

@@ -140,7 +140,7 @@ func unmarshalLocationElements(resp *http.Response, v reflect.Value, lowerCaseHe
prefix := field.Tag.Get("locationName")
err := unmarshalHeaderMap(m, resp.Header, prefix, lowerCaseHeaderMaps)
if err != nil {
awserr.New(request.ErrCodeSerialization, "failed to decode REST response", err)
return awserr.New(request.ErrCodeSerialization, "failed to decode REST response", err)
}
}
}

87
vendor/github.com/aws/aws-sdk-go/service/marketplacemetering/api.go сгенерированный поставляемый
Просмотреть файл

@@ -56,7 +56,7 @@ func (c *MarketplaceMetering) BatchMeterUsageRequest(input *BatchMeterUsageInput
// BatchMeterUsage API operation for AWSMarketplace Metering.
//
// BatchMeterUsage is called from a SaaS application listed on the AWS Marketplace
// BatchMeterUsage is called from a SaaS application listed on AWS Marketplace
// to post metering records for a set of customers.
//
// For identical requests, the API is idempotent; requests can be retried with
@@ -65,14 +65,26 @@ func (c *MarketplaceMetering) BatchMeterUsageRequest(input *BatchMeterUsageInput
// Every request to BatchMeterUsage is for one product. If you need to meter
// usage for multiple products, you must make multiple calls to BatchMeterUsage.
//
// Usage records are expected to be submitted as quickly as possible after the
// event that is being recorded, and are not accepted more than 6 hours after
// the event.
//
// BatchMeterUsage can process up to 25 UsageRecords at a time.
//
// A UsageRecord can optionally include multiple usage allocations, to provide
// customers with usagedata split into buckets by tags that you define (or allow
// the customer to define).
// customers with usage data split into buckets by tags that you define (or
// allow the customer to define).
//
// BatchMeterUsage returns a list of UsageRecordResult objects, showing the
// result for each UsageRecord, as well as a list of UnprocessedRecords, indicating
// errors in the service side that you should retry.
//
// BatchMeterUsage requests must be less than 1MB in size.
//
// For an example of using BatchMeterUsage, see BatchMeterUsage code example
// (https://docs.aws.amazon.com/marketplace/latest/userguide/saas-code-examples.html#saas-batchmeterusage-example)
// in the AWS Marketplace Seller Guide.
//
// Returns awserr.Error for service API and SDK errors. Use runtime type assertions
// with awserr.Error's Code and Message methods to get detailed information about
// the error.
@@ -104,7 +116,11 @@ func (c *MarketplaceMetering) BatchMeterUsageRequest(input *BatchMeterUsageInput
// You have metered usage for a CustomerIdentifier that does not exist.
//
// * TimestampOutOfBoundsException
// The timestamp value passed in the meterUsage() is out of allowed range.
// The timestamp value passed in the UsageRecord is out of allowed range.
//
// For BatchMeterUsage, if any of the records are outside of the allowed range,
// the entire batch is not processed. You must remove invalid records and try
// again.
//
// * ThrottlingException
// The calls to the API are throttled.
@@ -188,6 +204,10 @@ func (c *MarketplaceMetering) MeterUsageRequest(input *MeterUsageInput) (req *re
// customers with usage data split into buckets by tags that you define (or
// allow the customer to define).
//
// Usage records are expected to be submitted as quickly as possible after the
// event that is being recorded, and are not accepted more than 6 hours after
// the event.
//
// Returns awserr.Error for service API and SDK errors. Use runtime type assertions
// with awserr.Error's Code and Message methods to get detailed information about
// the error.
@@ -221,7 +241,11 @@ func (c *MarketplaceMetering) MeterUsageRequest(input *MeterUsageInput) (req *re
// AWS Region of the resource must match.
//
// * TimestampOutOfBoundsException
// The timestamp value passed in the meterUsage() is out of allowed range.
// The timestamp value passed in the UsageRecord is out of allowed range.
//
// For BatchMeterUsage, if any of the records are outside of the allowed range,
// the entire batch is not processed. You must remove invalid records and try
// again.
//
// * DuplicateRequestException
// A metering record has already been emitted by the same EC2 instance, ECS
@@ -313,7 +337,7 @@ func (c *MarketplaceMetering) RegisterUsageRequest(input *RegisterUsageInput) (r
// your paid software is subscribed to your product on AWS Marketplace, enabling
// you to guard against unauthorized use. Your container image that integrates
// with RegisterUsage is only required to guard against unauthorized use
// at container startup, as such a CustomerNotSubscribedException/PlatformNotSupportedException
// at container startup, as such a CustomerNotSubscribedException or PlatformNotSupportedException
// will only be thrown on the initial call to RegisterUsage. Subsequent calls
// from the same Amazon ECS task instance (e.g. task-id) or Amazon EKS pod
// will not throw a CustomerNotSubscribedException, even if the customer
@@ -440,7 +464,15 @@ func (c *MarketplaceMetering) ResolveCustomerRequest(input *ResolveCustomerInput
// ResolveCustomer is called by a SaaS application during the registration process.
// When a buyer visits your website during the registration process, the buyer
// submits a registration token through their browser. The registration token
// is resolved through this API to obtain a CustomerIdentifier and product code.
// is resolved through this API to obtain a CustomerIdentifier along with the
// CustomerAWSAccountId and ProductCode.
//
// The API needs to called from the seller account id used to publish the SaaS
// application to successfully resolve the token.
//
// For an example of using ResolveCustomer, see ResolveCustomer code example
// (https://docs.aws.amazon.com/marketplace/latest/userguide/saas-code-examples.html#saas-resolvecustomer-example)
// in the AWS Marketplace Seller Guide.
//
// Returns awserr.Error for service API and SDK errors. Use runtime type assertions
// with awserr.Error's Code and Message methods to get detailed information about
@@ -576,7 +608,8 @@ type BatchMeterUsageOutput struct {
_ struct{} `type:"structure"`
// Contains all UsageRecords processed by BatchMeterUsage. These records were
// either honored by AWS Marketplace Metering Service or were invalid.
// either honored by AWS Marketplace Metering Service or were invalid. Invalid
// records should be fixed before being resubmitted.
Results []*UsageRecordResult `type:"list"`
// Contains all UsageRecords that were not processed by BatchMeterUsage. This
@@ -1877,7 +1910,8 @@ type ResolveCustomerInput struct {
// When a buyer visits your website during the registration process, the buyer
// submits a registration token through the browser. The registration token
// is resolved to obtain a CustomerIdentifier and product code.
// is resolved to obtain a CustomerIdentifier along with the CustomerAWSAccountId
// and ProductCode.
//
// RegistrationToken is a required field
RegistrationToken *string `type:"string" required:"true"`
@@ -1921,10 +1955,14 @@ func (s *ResolveCustomerInput) SetRegistrationToken(v string) *ResolveCustomerIn
}
// The result of the ResolveCustomer operation. Contains the CustomerIdentifier
// and product code.
// along with the CustomerAWSAccountId and ProductCode.
type ResolveCustomerOutput struct {
_ struct{} `type:"structure"`
// The CustomerAWSAccountId provides the AWS account ID associated with the
// CustomerIdentifier for the individual customer.
CustomerAWSAccountId *string `min:"1" type:"string"`
// The CustomerIdentifier is used to identify an individual customer in your
// application. Calls to BatchMeterUsage require CustomerIdentifiers for each
// UsageRecord.
@@ -1954,6 +1992,12 @@ func (s ResolveCustomerOutput) GoString() string {
return s.String()
}
// SetCustomerAWSAccountId sets the CustomerAWSAccountId field's value.
func (s *ResolveCustomerOutput) SetCustomerAWSAccountId(v string) *ResolveCustomerOutput {
s.CustomerAWSAccountId = &v
return s
}
// SetCustomerIdentifier sets the CustomerIdentifier field's value.
func (s *ResolveCustomerOutput) SetCustomerIdentifier(v string) *ResolveCustomerOutput {
s.CustomerIdentifier = &v
@@ -2099,7 +2143,11 @@ func (s *ThrottlingException) RequestID() string {
return s.RespMetadata.RequestID
}
// The timestamp value passed in the meterUsage() is out of allowed range.
// The timestamp value passed in the UsageRecord is out of allowed range.
//
// For BatchMeterUsage, if any of the records are outside of the allowed range,
// the entire batch is not processed. You must remove invalid records and try
// again.
type TimestampOutOfBoundsException struct {
_ struct{} `type:"structure"`
RespMetadata protocol.ResponseMetadata `json:"-" xml:"-"`
@@ -2238,7 +2286,7 @@ func (s *UsageAllocation) SetTags(v []*Tag) *UsageAllocation {
// A UsageRecord indicates a quantity of usage for a given product, customer,
// dimension and time.
//
// Multiple requests with the same UsageRecords as input will be deduplicated
// Multiple requests with the same UsageRecords as input will be de-duplicated
// to prevent double charges.
type UsageRecord struct {
_ struct{} `type:"structure"`
@@ -2249,9 +2297,8 @@ type UsageRecord struct {
// CustomerIdentifier is a required field
CustomerIdentifier *string `min:"1" type:"string" required:"true"`
// During the process of registering a product on AWS Marketplace, up to eight
// dimensions are specified. These represent different units of value in your
// application.
// During the process of registering a product on AWS Marketplace, dimensions
// are specified. These represent different units of value in your application.
//
// Dimension is a required field
Dimension *string `min:"1" type:"string" required:"true"`
@@ -2372,9 +2419,13 @@ type UsageRecordResult struct {
//
// * Success- The UsageRecord was accepted and honored by BatchMeterUsage.
//
// * CustomerNotSubscribed- The CustomerIdentifier specified is not subscribed
// to your product. The UsageRecord was not honored. Future UsageRecords
// for this customer will fail until the customer subscribes to your product.
// * CustomerNotSubscribed- The CustomerIdentifier specified is not able
// to use your product. The UsageRecord was not honored. There are three
// causes for this result: The customer identifier is invalid. The customer
// identifier provided in the metering record does not have an active agreement
// or subscription with this product. Future UsageRecords for this customer
// will fail until the customer subscribes to your product. The customer's
// AWS account was suspended.
//
// * DuplicateRecord- Indicates that the UsageRecord was invalid and not
// honored. A previously metered UsageRecord had the same customer, dimension,

17
vendor/github.com/aws/aws-sdk-go/service/marketplacemetering/doc.go сгенерированный поставляемый
Просмотреть файл

@@ -9,25 +9,26 @@
// AWS Marketplace sellers can use this API to submit usage data for custom
// usage dimensions.
//
// For information on the permissions you need to use this API, see AWS Marketing
// For information on the permissions you need to use this API, see AWS Marketplace
// metering and entitlement API permissions (https://docs.aws.amazon.com/marketplace/latest/userguide/iam-user-policy-for-aws-marketplace-actions.html)
// in the AWS Marketplace Seller Guide.
//
// Submitting Metering Records
//
// * MeterUsage- Submits the metering record for a Marketplace product. MeterUsage
// is called from an EC2 instance or a container running on EKS or ECS.
// * MeterUsage - Submits the metering record for an AWS Marketplace product.
// MeterUsage is called from an EC2 instance or a container running on EKS
// or ECS.
//
// * BatchMeterUsage- Submits the metering record for a set of customers.
// * BatchMeterUsage - Submits the metering record for a set of customers.
// BatchMeterUsage is called from a software-as-a-service (SaaS) application.
//
// Accepting New Customers
//
// * ResolveCustomer- Called by a SaaS application during the registration
// * ResolveCustomer - Called by a SaaS application during the registration
// process. When a buyer visits your website during the registration process,
// the buyer submits a Registration Token through the browser. The Registration
// Token is resolved through this API to obtain a CustomerIdentifier and
// Product Code.
// Token is resolved through this API to obtain a CustomerIdentifier along
// with the CustomerAWSAccountId and ProductCode.
//
// Entitlement and Metering for Paid Container Products
//
@@ -43,7 +44,7 @@
// to verify that the SaaS metering records that you sent are accurate by searching
// for records with the eventName of BatchMeterUsage. You can also use CloudTrail
// to audit records over time. For more information, see the AWS CloudTrail
// User Guide (http://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-concepts.html) .
// User Guide (http://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-concepts.html).
//
// See https://docs.aws.amazon.com/goto/WebAPI/meteringmarketplace-2016-01-14 for more information on this service.
//

6
vendor/github.com/aws/aws-sdk-go/service/marketplacemetering/errors.go сгенерированный поставляемый
Просмотреть файл

@@ -123,7 +123,11 @@ const (
// ErrCodeTimestampOutOfBoundsException for service response error code
// "TimestampOutOfBoundsException".
//
// The timestamp value passed in the meterUsage() is out of allowed range.
// The timestamp value passed in the UsageRecord is out of allowed range.
//
// For BatchMeterUsage, if any of the records are outside of the allowed range,
// the entire batch is not processed. You must remove invalid records and try
// again.
ErrCodeTimestampOutOfBoundsException = "TimestampOutOfBoundsException"
)

4
vendor/github.com/aws/aws-sdk-go/service/sts/service.go сгенерированный поставляемый
Просмотреть файл

@@ -48,6 +48,10 @@ const (
// svc := sts.New(mySession, aws.NewConfig().WithRegion("us-west-2"))
func New(p client.ConfigProvider, cfgs ...*aws.Config) *STS {
c := p.ClientConfig(EndpointsID, cfgs...)
if c.SigningNameDerived || len(c.SigningName) == 0 {
c.SigningName = EndpointsID
// No Fallback
}
return newClient(*c.Config, c.Handlers, c.PartitionID, c.Endpoint, c.SigningRegion, c.SigningName, c.ResolvedRegion)
}

22
vendor/github.com/aymerick/douceur/LICENSE сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,22 @@
The MIT License (MIT)
Copyright (c) 2015 Aymerick JEHANNE
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

60
vendor/github.com/aymerick/douceur/css/declaration.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,60 @@
package css
import "fmt"
// Declaration represents a parsed style property
type Declaration struct {
Property string
Value string
Important bool
}
// NewDeclaration instanciates a new Declaration
func NewDeclaration() *Declaration {
return &Declaration{}
}
// Returns string representation of the Declaration
func (decl *Declaration) String() string {
return decl.StringWithImportant(true)
}
// StringWithImportant returns string representation with optional !important part
func (decl *Declaration) StringWithImportant(option bool) string {
result := fmt.Sprintf("%s: %s", decl.Property, decl.Value)
if option && decl.Important {
result += " !important"
}
result += ";"
return result
}
// Equal returns true if both Declarations are equals
func (decl *Declaration) Equal(other *Declaration) bool {
return (decl.Property == other.Property) && (decl.Value == other.Value) && (decl.Important == other.Important)
}
//
// DeclarationsByProperty
//
// DeclarationsByProperty represents sortable style declarations
type DeclarationsByProperty []*Declaration
// Implements sort.Interface
func (declarations DeclarationsByProperty) Len() int {
return len(declarations)
}
// Implements sort.Interface
func (declarations DeclarationsByProperty) Swap(i, j int) {
declarations[i], declarations[j] = declarations[j], declarations[i]
}
// Implements sort.Interface
func (declarations DeclarationsByProperty) Less(i, j int) bool {
return declarations[i].Property < declarations[j].Property
}

230
vendor/github.com/aymerick/douceur/css/rule.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,230 @@
package css
import (
"fmt"
"strings"
)
const (
indentSpace = 2
)
// RuleKind represents a Rule kind
type RuleKind int
// Rule kinds
const (
QualifiedRule RuleKind = iota
AtRule
)
// At Rules than have Rules inside their block instead of Declarations
var atRulesWithRulesBlock = []string{
"@document", "@font-feature-values", "@keyframes", "@media", "@supports",
}
// Rule represents a parsed CSS rule
type Rule struct {
Kind RuleKind
// At Rule name (eg: "@media")
Name string
// Raw prelude
Prelude string
// Qualified Rule selectors parsed from prelude
Selectors []string
// Style properties
Declarations []*Declaration
// At Rule embedded rules
Rules []*Rule
// Current rule embedding level
EmbedLevel int
}
// NewRule instanciates a new Rule
func NewRule(kind RuleKind) *Rule {
return &Rule{
Kind: kind,
}
}
// Returns string representation of rule kind
func (kind RuleKind) String() string {
switch kind {
case QualifiedRule:
return "Qualified Rule"
case AtRule:
return "At Rule"
default:
return "WAT"
}
}
// EmbedsRules returns true if this rule embeds another rules
func (rule *Rule) EmbedsRules() bool {
if rule.Kind == AtRule {
for _, atRuleName := range atRulesWithRulesBlock {
if rule.Name == atRuleName {
return true
}
}
}
return false
}
// Equal returns true if both rules are equals
func (rule *Rule) Equal(other *Rule) bool {
if (rule.Kind != other.Kind) ||
(rule.Prelude != other.Prelude) ||
(rule.Name != other.Name) {
return false
}
if (len(rule.Selectors) != len(other.Selectors)) ||
(len(rule.Declarations) != len(other.Declarations)) ||
(len(rule.Rules) != len(other.Rules)) {
return false
}
for i, sel := range rule.Selectors {
if sel != other.Selectors[i] {
return false
}
}
for i, decl := range rule.Declarations {
if !decl.Equal(other.Declarations[i]) {
return false
}
}
for i, rule := range rule.Rules {
if !rule.Equal(other.Rules[i]) {
return false
}
}
return true
}
// Diff returns a string representation of rules differences
func (rule *Rule) Diff(other *Rule) []string {
result := []string{}
if rule.Kind != other.Kind {
result = append(result, fmt.Sprintf("Kind: %s | %s", rule.Kind.String(), other.Kind.String()))
}
if rule.Prelude != other.Prelude {
result = append(result, fmt.Sprintf("Prelude: \"%s\" | \"%s\"", rule.Prelude, other.Prelude))
}
if rule.Name != other.Name {
result = append(result, fmt.Sprintf("Name: \"%s\" | \"%s\"", rule.Name, other.Name))
}
if len(rule.Selectors) != len(other.Selectors) {
result = append(result, fmt.Sprintf("Selectors: %v | %v", strings.Join(rule.Selectors, ", "), strings.Join(other.Selectors, ", ")))
} else {
for i, sel := range rule.Selectors {
if sel != other.Selectors[i] {
result = append(result, fmt.Sprintf("Selector: \"%s\" | \"%s\"", sel, other.Selectors[i]))
}
}
}
if len(rule.Declarations) != len(other.Declarations) {
result = append(result, fmt.Sprintf("Declarations Nb: %d | %d", len(rule.Declarations), len(other.Declarations)))
} else {
for i, decl := range rule.Declarations {
if !decl.Equal(other.Declarations[i]) {
result = append(result, fmt.Sprintf("Declaration: \"%s\" | \"%s\"", decl.String(), other.Declarations[i].String()))
}
}
}
if len(rule.Rules) != len(other.Rules) {
result = append(result, fmt.Sprintf("Rules Nb: %d | %d", len(rule.Rules), len(other.Rules)))
} else {
for i, rule := range rule.Rules {
if !rule.Equal(other.Rules[i]) {
result = append(result, fmt.Sprintf("Rule: \"%s\" | \"%s\"", rule.String(), other.Rules[i].String()))
}
}
}
return result
}
// Returns the string representation of a rule
func (rule *Rule) String() string {
result := ""
if rule.Kind == QualifiedRule {
for i, sel := range rule.Selectors {
if i != 0 {
result += ", "
}
result += sel
}
} else {
// AtRule
result += fmt.Sprintf("%s", rule.Name)
if rule.Prelude != "" {
if result != "" {
result += " "
}
result += fmt.Sprintf("%s", rule.Prelude)
}
}
if (len(rule.Declarations) == 0) && (len(rule.Rules) == 0) {
result += ";"
} else {
result += " {\n"
if rule.EmbedsRules() {
for _, subRule := range rule.Rules {
result += fmt.Sprintf("%s%s\n", rule.indent(), subRule.String())
}
} else {
for _, decl := range rule.Declarations {
result += fmt.Sprintf("%s%s\n", rule.indent(), decl.String())
}
}
result += fmt.Sprintf("%s}", rule.indentEndBlock())
}
return result
}
// Returns identation spaces for declarations and rules
func (rule *Rule) indent() string {
result := ""
for i := 0; i < ((rule.EmbedLevel + 1) * indentSpace); i++ {
result += " "
}
return result
}
// Returns identation spaces for end of block character
func (rule *Rule) indentEndBlock() string {
result := ""
for i := 0; i < (rule.EmbedLevel * indentSpace); i++ {
result += " "
}
return result
}

25
vendor/github.com/aymerick/douceur/css/stylesheet.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,25 @@
package css
// Stylesheet represents a parsed stylesheet
type Stylesheet struct {
Rules []*Rule
}
// NewStylesheet instanciate a new Stylesheet
func NewStylesheet() *Stylesheet {
return &Stylesheet{}
}
// Returns string representation of the Stylesheet
func (sheet *Stylesheet) String() string {
result := ""
for _, rule := range sheet.Rules {
if result != "" {
result += "\n"
}
result += rule.String()
}
return result
}

409
vendor/github.com/aymerick/douceur/parser/parser.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,409 @@
package parser
import (
"errors"
"fmt"
"regexp"
"strings"
"github.com/gorilla/css/scanner"
"github.com/aymerick/douceur/css"
)
const (
importantSuffixRegexp = `(?i)\s*!important\s*$`
)
var (
importantRegexp *regexp.Regexp
)
// Parser represents a CSS parser
type Parser struct {
scan *scanner.Scanner // Tokenizer
// Tokens parsed but not consumed yet
tokens []*scanner.Token
// Rule embedding level
embedLevel int
}
func init() {
importantRegexp = regexp.MustCompile(importantSuffixRegexp)
}
// NewParser instanciates a new parser
func NewParser(txt string) *Parser {
return &Parser{
scan: scanner.New(txt),
}
}
// Parse parses a whole stylesheet
func Parse(text string) (*css.Stylesheet, error) {
result, err := NewParser(text).ParseStylesheet()
if err != nil {
return nil, err
}
return result, nil
}
// ParseDeclarations parses CSS declarations
func ParseDeclarations(text string) ([]*css.Declaration, error) {
result, err := NewParser(text).ParseDeclarations()
if err != nil {
return nil, err
}
return result, nil
}
// ParseStylesheet parses a stylesheet
func (parser *Parser) ParseStylesheet() (*css.Stylesheet, error) {
result := css.NewStylesheet()
// Parse BOM
if _, err := parser.parseBOM(); err != nil {
return result, err
}
// Parse list of rules
rules, err := parser.ParseRules()
if err != nil {
return result, err
}
result.Rules = rules
return result, nil
}
// ParseRules parses a list of rules
func (parser *Parser) ParseRules() ([]*css.Rule, error) {
result := []*css.Rule{}
inBlock := false
if parser.tokenChar("{") {
// parsing a block of rules
inBlock = true
parser.embedLevel++
parser.shiftToken()
}
for parser.tokenParsable() {
if parser.tokenIgnorable() {
parser.shiftToken()
} else if parser.tokenChar("}") {
if !inBlock {
errMsg := fmt.Sprintf("Unexpected } character: %s", parser.nextToken().String())
return result, errors.New(errMsg)
}
parser.shiftToken()
parser.embedLevel--
// finished
break
} else {
rule, err := parser.ParseRule()
if err != nil {
return result, err
}
rule.EmbedLevel = parser.embedLevel
result = append(result, rule)
}
}
return result, parser.err()
}
// ParseRule parses a rule
func (parser *Parser) ParseRule() (*css.Rule, error) {
if parser.tokenAtKeyword() {
return parser.parseAtRule()
}
return parser.parseQualifiedRule()
}
// ParseDeclarations parses a list of declarations
func (parser *Parser) ParseDeclarations() ([]*css.Declaration, error) {
result := []*css.Declaration{}
if parser.tokenChar("{") {
parser.shiftToken()
}
for parser.tokenParsable() {
if parser.tokenIgnorable() {
parser.shiftToken()
} else if parser.tokenChar("}") {
// end of block
parser.shiftToken()
break
} else {
declaration, err := parser.ParseDeclaration()
if err != nil {
return result, err
}
result = append(result, declaration)
}
}
return result, parser.err()
}
// ParseDeclaration parses a declaration
func (parser *Parser) ParseDeclaration() (*css.Declaration, error) {
result := css.NewDeclaration()
curValue := ""
for parser.tokenParsable() {
if parser.tokenChar(":") {
result.Property = strings.TrimSpace(curValue)
curValue = ""
parser.shiftToken()
} else if parser.tokenChar(";") || parser.tokenChar("}") {
if result.Property == "" {
errMsg := fmt.Sprintf("Unexpected ; character: %s", parser.nextToken().String())
return result, errors.New(errMsg)
}
if importantRegexp.MatchString(curValue) {
result.Important = true
curValue = importantRegexp.ReplaceAllString(curValue, "")
}
result.Value = strings.TrimSpace(curValue)
if parser.tokenChar(";") {
parser.shiftToken()
}
// finished
break
} else {
token := parser.shiftToken()
curValue += token.Value
}
}
// log.Printf("[parsed] Declaration: %s", result.String())
return result, parser.err()
}
// Parse an At Rule
func (parser *Parser) parseAtRule() (*css.Rule, error) {
// parse rule name (eg: "@import")
token := parser.shiftToken()
result := css.NewRule(css.AtRule)
result.Name = token.Value
for parser.tokenParsable() {
if parser.tokenChar(";") {
parser.shiftToken()
// finished
break
} else if parser.tokenChar("{") {
if result.EmbedsRules() {
// parse rules block
rules, err := parser.ParseRules()
if err != nil {
return result, err
}
result.Rules = rules
} else {
// parse declarations block
declarations, err := parser.ParseDeclarations()
if err != nil {
return result, err
}
result.Declarations = declarations
}
// finished
break
} else {
// parse prelude
prelude, err := parser.parsePrelude()
if err != nil {
return result, err
}
result.Prelude = prelude
}
}
// log.Printf("[parsed] Rule: %s", result.String())
return result, parser.err()
}
// Parse a Qualified Rule
func (parser *Parser) parseQualifiedRule() (*css.Rule, error) {
result := css.NewRule(css.QualifiedRule)
for parser.tokenParsable() {
if parser.tokenChar("{") {
if result.Prelude == "" {
errMsg := fmt.Sprintf("Unexpected { character: %s", parser.nextToken().String())
return result, errors.New(errMsg)
}
// parse declarations block
declarations, err := parser.ParseDeclarations()
if err != nil {
return result, err
}
result.Declarations = declarations
// finished
break
} else {
// parse prelude
prelude, err := parser.parsePrelude()
if err != nil {
return result, err
}
result.Prelude = prelude
}
}
result.Selectors = strings.Split(result.Prelude, ",")
for i, sel := range result.Selectors {
result.Selectors[i] = strings.TrimSpace(sel)
}
// log.Printf("[parsed] Rule: %s", result.String())
return result, parser.err()
}
// Parse Rule prelude
func (parser *Parser) parsePrelude() (string, error) {
result := ""
for parser.tokenParsable() && !parser.tokenEndOfPrelude() {
token := parser.shiftToken()
result += token.Value
}
result = strings.TrimSpace(result)
// log.Printf("[parsed] prelude: %s", result)
return result, parser.err()
}
// Parse BOM
func (parser *Parser) parseBOM() (bool, error) {
if parser.nextToken().Type == scanner.TokenBOM {
parser.shiftToken()
return true, nil
}
return false, parser.err()
}
// Returns next token without removing it from tokens buffer
func (parser *Parser) nextToken() *scanner.Token {
if len(parser.tokens) == 0 {
// fetch next token
nextToken := parser.scan.Next()
// log.Printf("[token] %s => %v", nextToken.Type.String(), nextToken.Value)
// queue it
parser.tokens = append(parser.tokens, nextToken)
}
return parser.tokens[0]
}
// Returns next token and remove it from the tokens buffer
func (parser *Parser) shiftToken() *scanner.Token {
var result *scanner.Token
result, parser.tokens = parser.tokens[0], parser.tokens[1:]
return result
}
// Returns tokenizer error, or nil if no error
func (parser *Parser) err() error {
if parser.tokenError() {
token := parser.nextToken()
return fmt.Errorf("Tokenizer error: %s", token.String())
}
return nil
}
// Returns true if next token is Error
func (parser *Parser) tokenError() bool {
return parser.nextToken().Type == scanner.TokenError
}
// Returns true if next token is EOF
func (parser *Parser) tokenEOF() bool {
return parser.nextToken().Type == scanner.TokenEOF
}
// Returns true if next token is a whitespace
func (parser *Parser) tokenWS() bool {
return parser.nextToken().Type == scanner.TokenS
}
// Returns true if next token is a comment
func (parser *Parser) tokenComment() bool {
return parser.nextToken().Type == scanner.TokenComment
}
// Returns true if next token is a CDO or a CDC
func (parser *Parser) tokenCDOorCDC() bool {
switch parser.nextToken().Type {
case scanner.TokenCDO, scanner.TokenCDC:
return true
default:
return false
}
}
// Returns true if next token is ignorable
func (parser *Parser) tokenIgnorable() bool {
return parser.tokenWS() || parser.tokenComment() || parser.tokenCDOorCDC()
}
// Returns true if next token is parsable
func (parser *Parser) tokenParsable() bool {
return !parser.tokenEOF() && !parser.tokenError()
}
// Returns true if next token is an At Rule keyword
func (parser *Parser) tokenAtKeyword() bool {
return parser.nextToken().Type == scanner.TokenAtKeyword
}
// Returns true if next token is given character
func (parser *Parser) tokenChar(value string) bool {
token := parser.nextToken()
return (token.Type == scanner.TokenChar) && (token.Value == value)
}
// Returns true if next token marks the end of a prelude
func (parser *Parser) tokenEndOfPrelude() bool {
return parser.tokenChar(";") || parser.tokenChar("{")
}

12
vendor/github.com/getsentry/sentry-go/.craft.yml сгенерированный поставляемый
Просмотреть файл

@@ -1,11 +1,6 @@
minVersion: '0.9.2'
github:
owner: getsentry
repo: sentry-go
minVersion: 0.23.1
preReleaseCommand: bash scripts/craft-pre-release.sh
changelogPolicy: simple
statusProvider:
name: github
artifactProvider:
name: none
targets:
@@ -13,6 +8,5 @@ targets:
includeNames: /none/
tagPrefix: v
- name: registry
type: sdk
config:
canonical: "github:getsentry/sentry-go"
sdks:
github:getsentry/sentry-go:

12
vendor/github.com/getsentry/sentry-go/CHANGELOG.md сгенерированный поставляемый
Просмотреть файл

@@ -1,5 +1,17 @@
# Changelog
## v0.12.0
- feat: Automatic Release detection (#363, #369, #386, #400)
- fix: Do not change Hub.lastEventID for transactions (#379)
- fix: Do not clear LastEventID when events are dropped (#382)
- Updates to documentation (#366, #385)
_NOTE:_
This version drops support for Go 1.14, however no changes have been made that would make the SDK not work with Go 1.14. The currently supported Go versions are the last 3 stable releases: 1.15, 1.16 and 1.17.
There are two behavior changes related to `LastEventID`, both of which were intended to align the behavior of the Sentry Go SDK with other Sentry SDKs.
The new [automatic release detection feature](https://github.com/getsentry/sentry-go/issues/335) makes it easier to use Sentry and separate events per release without requiring extra work from users. We intend to improve this functionality in a future release by utilizing information that will be available in runtime starting with Go 1.18. The tracking issue is [#401](https://github.com/getsentry/sentry-go/issues/401).
## v0.11.0
- feat(transports): Category-based Rate Limiting ([#354](https://github.com/getsentry/sentry-go/pull/354))

391
vendor/github.com/getsentry/sentry-go/MIGRATION.md сгенерированный поставляемый
Просмотреть файл

@@ -1,392 +1,3 @@
# `raven-go` to `sentry-go` Migration Guide
## Installation
raven-go
```go
go get github.com/getsentry/raven-go
```
sentry-go
```go
go get github.com/getsentry/sentry-go@v0.0.1
```
## Configuration
raven-go
```go
import "github.com/getsentry/raven-go"
func main() {
raven.SetDSN("___PUBLIC_DSN___")
}
```
sentry-go
```go
import (
"fmt"
"github.com/getsentry/sentry-go"
)
func main() {
err := sentry.Init(sentry.ClientOptions{
Dsn: "___PUBLIC_DSN___",
})
if err != nil {
fmt.Printf("Sentry initialization failed: %v\n", err)
}
}
```
raven-go
```go
SetDSN()
SetDefaultLoggerName()
SetDebug()
SetEnvironment()
SetRelease()
SetSampleRate()
SetIgnoreErrors()
SetIncludePaths()
```
sentry-go
```go
sentry.Init(sentry.ClientOptions{
Dsn: "___PUBLIC_DSN___",
DebugWriter: os.Stderr,
Debug: true,
Environment: "environment",
Release: "release",
SampleRate: 0.5,
// IgnoreErrors: TBD,
// IncludePaths: TBD
})
```
Available options: see [Configuration](https://docs.sentry.io/platforms/go/config/) section.
### Providing SSL Certificates
By default, TLS uses the host's root CA set. If you don't have `ca-certificates` (which should be your go-to way of fixing the issue of missing certificates) and want to use `gocertifi` instead, you can provide pre-loaded cert files as one of the options to the `sentry.Init` call:
```go
package main
import (
"log"
"github.com/certifi/gocertifi"
"github.com/getsentry/sentry-go"
)
sentryClientOptions := sentry.ClientOptions{
Dsn: "___PUBLIC_DSN___",
}
rootCAs, err := gocertifi.CACerts()
if err != nil {
log.Println("Couldn't load CA Certificates: %v\n", err)
} else {
sentryClientOptions.CaCerts = rootCAs
}
sentry.Init(sentryClientOptions)
```
## Usage
### Capturing Errors
raven-go
```go
f, err := os.Open("filename.ext")
if err != nil {
raven.CaptureError(err, nil)
}
```
sentry-go
```go
f, err := os.Open("filename.ext")
if err != nil {
sentry.CaptureException(err)
}
```
### Capturing Panics
raven-go
```go
raven.CapturePanic(func() {
// do all of the scary things here
}, nil)
```
sentry-go
```go
func() {
defer sentry.Recover()
// do all of the scary things here
}()
```
### Capturing Messages
raven-go
```go
raven.CaptureMessage("Something bad happened and I would like to know about that")
```
sentry-go
```go
sentry.CaptureMessage("Something bad happened and I would like to know about that")
```
### Capturing Events
raven-go
```go
packet := &raven.Packet{
Message: "Hand-crafted event",
Extra: &raven.Extra{
"runtime.Version": runtime.Version(),
"runtime.NumCPU": runtime.NumCPU(),
},
}
raven.Capture(packet)
```
sentry-go
```go
event := &sentry.NewEvent()
event.Message = "Hand-crafted event"
event.Extra["runtime.Version"] = runtime.Version()
event.Extra["runtime.NumCPU"] = runtime.NumCPU()
sentry.CaptureEvent(event)
```
### Additional Data
See Context section.
### Event Sampling
raven-go
```go
raven.SetSampleRate(0.25)
```
sentry-go
```go
sentry.Init(sentry.ClientOptions{
SampleRate: 0.25,
})
```
### Awaiting the response (not recommended)
```go
raven.CaptureMessageAndWait("Something bad happened and I would like to know about that")
```
sentry-go
```go
sentry.CaptureMessage("Something bad happened and I would like to know about that")
if sentry.Flush(time.Second * 2) {
// event delivered
} else {
// timeout reached
}
```
## Context
### Per-event
raven-go
```go
raven.CaptureError(err, map[string]string{"browser": "Firefox"}, &raven.Http{
Method: "GET",
URL: "https://example.com/raven-go"
})
```
sentry-go
```go
sentry.WithScope(func(scope *sentry.Scope) {
scope.SetTag("browser", "Firefox")
scope.SetContext("Request", map[string]string{
"Method": "GET",
"URL": "https://example.com/raven-go",
})
sentry.CaptureException(err)
})
```
### Globally
#### SetHttpContext
raven-go
```go
raven.SetHttpContext(&raven.Http{
Method: "GET",
URL: "https://example.com/raven-go",
})
```
sentry-go
```go
sentry.ConfigureScope(func(scope *sentry.Scope) {
scope.SetContext("Request", map[string]string{
"Method": "GET",
"URL": "https://example.com/raven-go",
})
})
```
#### SetTagsContext
raven-go
```go
t := map[string]string{"day": "Friday", "sport": "Weightlifting"}
raven.SetTagsContext(map[string]string{"day": "Friday", "sport": "Weightlifting"})
```
sentry-go
```go
sentry.ConfigureScope(func(scope *sentry.Scope) {
scope.SetTags(map[string]string{"day": "Friday", "sport": "Weightlifting"})
})
```
#### SetUserContext
raven-go
```go
raven.SetUserContext(&raven.User{
ID: "1337",
Username: "kamilogorek",
Email: "kamil@sentry.io",
IP: "127.0.0.1",
})
```
sentry-go
```go
sentry.ConfigureScope(func(scope *sentry.Scope) {
scope.SetUser(sentry.User{
ID: "1337",
Username: "kamilogorek",
Email: "kamil@sentry.io",
IPAddress: "127.0.0.1",
})
})
```
#### ClearContext
raven-go
```go
raven.ClearContext()
```
sentry-go
```go
sentry.ConfigureScope(func(scope *sentry.Scope) {
scope.Clear()
})
```
#### WrapWithExtra
raven-go
```go
path := "filename.ext"
f, err := os.Open(path)
if err != nil {
err = raven.WrapWithExtra(err, map[string]string{"path": path, "cwd": os.Getwd()}
raven.CaptureError(err, nil)
}
```
sentry-go
```go
// use `sentry.WithScope`, see "Context / Per-event Section"
path := "filename.ext"
f, err := os.Open(path)
if err != nil {
sentry.WithScope(func(scope *sentry.Scope) {
sentry.SetExtras(map[string]interface{}{"path": path, "cwd": os.Getwd())
sentry.CaptureException(err)
})
}
```
## Integrations
### net/http
raven-go
```go
mux := http.NewServeMux
http.Handle("/", raven.Recoverer(mux))
// or
func root(w http.ResponseWriter, r *http.Request) {}
http.HandleFunc("/", raven.RecoveryHandler(root))
```
sentry-go
```go
sentryHandler := sentryhttp.New(sentryhttp.Options{
Repanic: false,
WaitForDelivery: true,
})
mux := http.NewServeMux
http.Handle("/", sentryHandler.Handle(mux))
// or
func root(w http.ResponseWriter, r *http.Request) {}
http.HandleFunc("/", sentryHandler.HandleFunc(root))
```
A [`raven-go` to `sentry-go` migration guide](https://docs.sentry.io/platforms/go/migration/) is available at the official Sentry documentation site.

24
vendor/github.com/getsentry/sentry-go/client.go сгенерированный поставляемый
Просмотреть файл

@@ -145,6 +145,28 @@ type ClientOptions struct {
// The server name to be reported.
ServerName string
// The release to be sent with events.
//
// Some Sentry features are built around releases, and, thus, reporting
// events with a non-empty release improves the product experience. See
// https://docs.sentry.io/product/releases/.
//
// If Release is not set, the SDK will try to derive a default value
// from environment variables or the Git repository in the working
// directory.
//
// If you distribute a compiled binary, it is recommended to set the
// Release value explicitly at build time. As an example, you can use:
//
// go build -ldflags='-X main.release=VALUE'
//
// That will set the value of a predeclared variable 'release' in the
// 'main' package to 'VALUE'. Then, use that variable when initializing
// the SDK:
//
// sentry.Init(ClientOptions{Release: release})
//
// See https://golang.org/cmd/go/ and https://golang.org/cmd/link/ for
// the official documentation of -ldflags and -X, respectively.
Release string
// The dist to be sent with events.
Dist string
@@ -208,7 +230,7 @@ func NewClient(options ClientOptions) (*Client, error) {
}
if options.Release == "" {
options.Release = os.Getenv("SENTRY_RELEASE")
options.Release = defaultRelease()
}
if options.Environment == "" {

9
vendor/github.com/getsentry/sentry-go/go.mod сгенерированный поставляемый
Просмотреть файл

@@ -1,6 +1,6 @@
module github.com/getsentry/sentry-go
go 1.14
go 1.15
require (
github.com/ajg/form v1.5.1 // indirect
@@ -14,7 +14,8 @@ require (
github.com/imkira/go-interpol v1.1.0 // indirect
github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 // indirect
github.com/kataras/iris/v12 v12.1.8
github.com/labstack/echo/v4 v4.1.11
github.com/labstack/echo/v4 v4.5.0
github.com/mattn/go-colorable v0.1.11 // indirect
github.com/moul/http2curl v1.0.0 // indirect
github.com/onsi/ginkgo v1.10.3 // indirect
github.com/onsi/gomega v1.7.1 // indirect
@@ -31,4 +32,8 @@ require (
github.com/yudai/gojsondiff v1.0.0 // indirect
github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect
github.com/yudai/pp v2.0.1+incompatible // indirect
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
golang.org/x/net v0.0.0-20211008194852-3b03d305991f // indirect
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac
golang.org/x/text v0.3.7 // indirect
)

48
vendor/github.com/getsentry/sentry-go/go.sum сгенерированный поставляемый
Просмотреть файл

@@ -24,8 +24,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o=
@@ -52,6 +50,8 @@ github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AE
github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -106,17 +106,21 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/labstack/echo/v4 v4.1.11 h1:z0BZoArY4FqdpUEl+wlHp4hnr/oSR6MTmQmv8OHSoww=
github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g=
github.com/labstack/echo/v4 v4.5.0 h1:JXk6H5PAw9I3GwizqUHhYyS4f45iyGebR/c1xNCeOCY=
github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y=
github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0=
github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-colorable v0.1.11 h1:nQ+aFkoE2TMGc0b68U2OKSexC+eq46+XwZzWXHRmPYs=
github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg=
github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8=
github.com/microcosm-cc/bluemonday v1.0.2 h1:5lPfLTTAvAbtS0VqT+94yOtFnGfUWYyx0+iToC3Os3s=
@@ -183,8 +187,9 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.6.0 h1:uWF8lgKmeaIewWVPwi4GRq2P6+R46IgYZdxWtM+GtEY=
github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w=
github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8=
github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4=
github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
@@ -204,8 +209,10 @@ github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZ
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876 h1:sKJQZMuxjOAR/Uo2LBfU90onWEf1dF4C+0hPJCc9Mpc=
golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -214,8 +221,11 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 h1:efeOvDhwQ29Dj3SdAV/MJf8oukgn+8D8WgaCaRMchF8=
golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20211008194852-3b03d305991f h1:1scJEYZBaF48BaG6tYbtxmLcXqwYGSfGcMoStTqkkIw=
golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -224,11 +234,27 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ=
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac h1:oN6lz7iLW/YC7un8pq+9bOLyXrprv2+DKfkJY+2LJJw=
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 h1:Hir2P/De0WpUhtrKGGjvSb2YxUgyZ7EFOSLIcSSpiwE=
golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=

33
vendor/github.com/getsentry/sentry-go/hub.go сгенерированный поставляемый
Просмотреть файл

@@ -84,7 +84,18 @@ func CurrentHub() *Hub {
return currentHub
}
// LastEventID returns an ID of last captured event for the current Hub.
// LastEventID returns the ID of the last event (error or message) captured
// through the hub and sent to the underlying transport.
//
// Transactions and events dropped by sampling or event processors do not change
// the last event ID.
//
// LastEventID is a convenience method to cover use cases in which errors are
// captured indirectly and the ID is needed. For example, it can be used as part
// of an HTTP middleware to log the ID of the last error, if any.
//
// For more flexibility, consider instead using the ClientOptions.BeforeSend
// function or event processors.
func (hub *Hub) LastEventID() EventID {
hub.mu.RLock()
defer hub.mu.RUnlock()
@@ -212,12 +223,10 @@ func (hub *Hub) CaptureEvent(event *Event) *EventID {
}
eventID := client.CaptureEvent(event, nil, scope)
hub.mu.Lock()
defer hub.mu.Unlock()
if eventID != nil {
if event.Type != transactionType && eventID != nil {
hub.mu.Lock()
hub.lastEventID = *eventID
} else {
hub.lastEventID = ""
hub.mu.Unlock()
}
return eventID
}
@@ -232,12 +241,10 @@ func (hub *Hub) CaptureMessage(message string) *EventID {
}
eventID := client.CaptureMessage(message, nil, scope)
hub.mu.Lock()
defer hub.mu.Unlock()
if eventID != nil {
hub.mu.Lock()
hub.lastEventID = *eventID
} else {
hub.lastEventID = ""
hub.mu.Unlock()
}
return eventID
}
@@ -252,12 +259,10 @@ func (hub *Hub) CaptureException(exception error) *EventID {
}
eventID := client.CaptureException(exception, &EventHint{OriginalException: exception}, scope)
hub.mu.Lock()
defer hub.mu.Unlock()
if eventID != nil {
hub.mu.Lock()
hub.lastEventID = *eventID
} else {
hub.lastEventID = ""
hub.mu.Unlock()
}
return eventID
}

2
vendor/github.com/getsentry/sentry-go/sentry.go сгенерированный поставляемый
Просмотреть файл

@@ -6,7 +6,7 @@ import (
)
// Version is the version of the SDK.
const Version = "0.11.0"
const Version = "0.12.0"
// apiVersion is the minimum version of the Sentry API compatible with the
// sentry-go SDK.

13
vendor/github.com/getsentry/sentry-go/stacktrace.go сгенерированный поставляемый
Просмотреть файл

@@ -160,9 +160,16 @@ func extractXErrorsPC(err error) []uintptr {
// Frame represents a function call and it's metadata. Frames are associated
// with a Stacktrace.
type Frame struct {
Function string `json:"function,omitempty"`
Symbol string `json:"symbol,omitempty"`
Module string `json:"module,omitempty"`
Function string `json:"function,omitempty"`
Symbol string `json:"symbol,omitempty"`
// Module is, despite the name, the Sentry protocol equivalent of a Go
// package's import path.
Module string `json:"module,omitempty"`
// Package is not used for Go stack trace frames. In other platforms it
// refers to a container where the Module can be found. For example, a
// Java JAR, a .NET Assembly, or a native dynamic library.
// It exists for completeness, allowing the construction and reporting
// of custom event payloads.
Package string `json:"package,omitempty"`
Filename string `json:"filename,omitempty"`
AbsPath string `json:"abs_path,omitempty"`

52
vendor/github.com/getsentry/sentry-go/util.go сгенерированный поставляемый
Просмотреть файл

@@ -6,7 +6,10 @@ import (
"encoding/json"
"fmt"
"os"
"strings"
"time"
exec "golang.org/x/sys/execabs"
)
func uuid() string {
@@ -37,3 +40,52 @@ func prettyPrint(data interface{}) {
dbg, _ := json.MarshalIndent(data, "", " ")
fmt.Println(string(dbg))
}
// defaultRelease attempts to guess a default release for the currently running
// program.
func defaultRelease() (release string) {
// Return first non-empty environment variable known to hold release info, if any.
envs := []string{
"SENTRY_RELEASE",
"HEROKU_SLUG_COMMIT",
"SOURCE_VERSION",
"CODEBUILD_RESOLVED_SOURCE_VERSION",
"CIRCLE_SHA1",
"GAE_DEPLOYMENT_ID",
"GITHUB_SHA", // GitHub Actions - https://help.github.com/en/actions
"COMMIT_REF", // Netlify - https://docs.netlify.com/
"VERCEL_GIT_COMMIT_SHA", // Vercel - https://vercel.com/
"ZEIT_GITHUB_COMMIT_SHA", // Zeit (now known as Vercel)
"ZEIT_GITLAB_COMMIT_SHA",
"ZEIT_BITBUCKET_COMMIT_SHA",
}
for _, e := range envs {
if release = os.Getenv(e); release != "" {
Logger.Printf("Using release from environment variable %s: %s", e, release)
return release
}
}
// Derive a version string from Git. Example outputs:
// v1.0.1-0-g9de4
// v2.0-8-g77df-dirty
// 4f72d7
cmd := exec.Command("git", "describe", "--long", "--always", "--dirty")
b, err := cmd.Output()
if err != nil {
// Either Git is not available or the current directory is not a
// Git repository.
var s strings.Builder
fmt.Fprintf(&s, "Release detection failed: %v", err)
if err, ok := err.(*exec.ExitError); ok && len(err.Stderr) > 0 {
fmt.Fprintf(&s, ": %s", err.Stderr)
}
Logger.Print(s.String())
Logger.Print("Some Sentry features will not be available. See https://docs.sentry.io/product/releases/.")
Logger.Print("To stop seeing this message, pass a Release to sentry.Init or set the SENTRY_RELEASE environment variable.")
return ""
}
release = strings.TrimSpace(string(b))
Logger.Printf("Using release from Git: %s", release)
return release
}

8
vendor/github.com/gopherjs/gopherjs/js/js.go сгенерированный поставляемый
Просмотреть файл

@@ -97,7 +97,13 @@ func (err *Error) Stack() string {
// Global gives JavaScript's global object ("window" for browsers and "GLOBAL" for Node.js).
var Global *Object
// Module gives the value of the "module" variable set by Node.js. Hint: Set a module export with 'js.Module.Get("exports").Set("exportName", ...)'.
// Module gives the value of the "module" variable set by Node.js. Hint: Set a
// module export with 'js.Module.Get("exports").Set("exportName", ...)'.
//
// Note that js.Module is only defined in runtimes which support CommonJS
// modules (https://nodejs.org/api/modules.html). NodeJS supports it natively,
// but in browsers it can only be used if GopherJS output is passed through a
// bundler which implements CommonJS (for example, webpack or esbuild).
var Module *Object
// Undefined gives the JavaScript value "undefined".

27
vendor/github.com/gorilla/css/LICENSE сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,27 @@
Copyright (c) 2013, Gorilla web toolkit
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright notice, this
list of conditions and the following disclaimer in the documentation and/or
other materials provided with the distribution.
Neither the name of the {organization} nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

33
vendor/github.com/gorilla/css/scanner/doc.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,33 @@
// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
/*
Package gorilla/css/scanner generates tokens for a CSS3 input.
It follows the CSS3 specification located at:
http://www.w3.org/TR/css3-syntax/
To use it, create a new scanner for a given CSS string and call Next() until
the token returned has type TokenEOF or TokenError:
s := scanner.New(myCSS)
for {
token := s.Next()
if token.Type == scanner.TokenEOF || token.Type == scanner.TokenError {
break
}
// Do something with the token...
}
Following the CSS3 specification, an error can only occur when the scanner
finds an unclosed quote or unclosed comment. In these cases the text becomes
"untokenizable". Everything else is tokenizable and it is up to a parser
to make sense of the token stream (or ignore nonsensical token sequences).
Note: the scanner doesn't perform lexical analysis or, in other words, it
doesn't care about the token context. It is intended to be used by a
lexer or parser.
*/
package scanner

356
vendor/github.com/gorilla/css/scanner/scanner.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,356 @@
// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package scanner
import (
"fmt"
"regexp"
"strings"
"unicode"
"unicode/utf8"
)
// tokenType identifies the type of lexical tokens.
type tokenType int
// String returns a string representation of the token type.
func (t tokenType) String() string {
return tokenNames[t]
}
// Token represents a token and the corresponding string.
type Token struct {
Type tokenType
Value string
Line int
Column int
}
// String returns a string representation of the token.
func (t *Token) String() string {
if len(t.Value) > 10 {
return fmt.Sprintf("%s (line: %d, column: %d): %.10q...",
t.Type, t.Line, t.Column, t.Value)
}
return fmt.Sprintf("%s (line: %d, column: %d): %q",
t.Type, t.Line, t.Column, t.Value)
}
// All tokens -----------------------------------------------------------------
// The complete list of tokens in CSS3.
const (
// Scanner flags.
TokenError tokenType = iota
TokenEOF
// From now on, only tokens from the CSS specification.
TokenIdent
TokenAtKeyword
TokenString
TokenHash
TokenNumber
TokenPercentage
TokenDimension
TokenURI
TokenUnicodeRange
TokenCDO
TokenCDC
TokenS
TokenComment
TokenFunction
TokenIncludes
TokenDashMatch
TokenPrefixMatch
TokenSuffixMatch
TokenSubstringMatch
TokenChar
TokenBOM
)
// tokenNames maps tokenType's to their names. Used for conversion to string.
var tokenNames = map[tokenType]string{
TokenError: "error",
TokenEOF: "EOF",
TokenIdent: "IDENT",
TokenAtKeyword: "ATKEYWORD",
TokenString: "STRING",
TokenHash: "HASH",
TokenNumber: "NUMBER",
TokenPercentage: "PERCENTAGE",
TokenDimension: "DIMENSION",
TokenURI: "URI",
TokenUnicodeRange: "UNICODE-RANGE",
TokenCDO: "CDO",
TokenCDC: "CDC",
TokenS: "S",
TokenComment: "COMMENT",
TokenFunction: "FUNCTION",
TokenIncludes: "INCLUDES",
TokenDashMatch: "DASHMATCH",
TokenPrefixMatch: "PREFIXMATCH",
TokenSuffixMatch: "SUFFIXMATCH",
TokenSubstringMatch: "SUBSTRINGMATCH",
TokenChar: "CHAR",
TokenBOM: "BOM",
}
// Macros and productions -----------------------------------------------------
// http://www.w3.org/TR/css3-syntax/#tokenization
var macroRegexp = regexp.MustCompile(`\{[a-z]+\}`)
// macros maps macro names to patterns to be expanded.
var macros = map[string]string{
// must be escaped: `\.+*?()|[]{}^$`
"ident": `-?{nmstart}{nmchar}*`,
"name": `{nmchar}+`,
"nmstart": `[a-zA-Z_]|{nonascii}|{escape}`,
"nonascii": "[\u0080-\uD7FF\uE000-\uFFFD\U00010000-\U0010FFFF]",
"unicode": `\\[0-9a-fA-F]{1,6}{wc}?`,
"escape": "{unicode}|\\\\[\u0020-\u007E\u0080-\uD7FF\uE000-\uFFFD\U00010000-\U0010FFFF]",
"nmchar": `[a-zA-Z0-9_-]|{nonascii}|{escape}`,
"num": `[0-9]*\.[0-9]+|[0-9]+`,
"string": `"(?:{stringchar}|')*"|'(?:{stringchar}|")*'`,
"stringchar": `{urlchar}|[ ]|\\{nl}`,
"nl": `[\n\r\f]|\r\n`,
"w": `{wc}*`,
"wc": `[\t\n\f\r ]`,
// urlchar should accept [(ascii characters minus those that need escaping)|{nonascii}|{escape}]
// ASCII characters range = `[\u0020-\u007e]`
// Skip space \u0020 = `[\u0021-\u007e]`
// Skip quotation mark \0022 = `[\u0021\u0023-\u007e]`
// Skip apostrophe \u0027 = `[\u0021\u0023-\u0026\u0028-\u007e]`
// Skip reverse solidus \u005c = `[\u0021\u0023-\u0026\u0028-\u005b\u005d\u007e]`
// Finally, the left square bracket (\u005b) and right (\u005d) needs escaping themselves
"urlchar": "[\u0021\u0023-\u0026\u0028-\\\u005b\\\u005d-\u007E]|{nonascii}|{escape}",
}
// productions maps the list of tokens to patterns to be expanded.
var productions = map[tokenType]string{
// Unused regexps (matched using other methods) are commented out.
TokenIdent: `{ident}`,
TokenAtKeyword: `@{ident}`,
TokenString: `{string}`,
TokenHash: `#{name}`,
TokenNumber: `{num}`,
TokenPercentage: `{num}%`,
TokenDimension: `{num}{ident}`,
TokenURI: `url\({w}(?:{string}|{urlchar}*?){w}\)`,
TokenUnicodeRange: `U\+[0-9A-F\?]{1,6}(?:-[0-9A-F]{1,6})?`,
//TokenCDO: `<!--`,
TokenCDC: `-->`,
TokenS: `{wc}+`,
TokenComment: `/\*[^\*]*[\*]+(?:[^/][^\*]*[\*]+)*/`,
TokenFunction: `{ident}\(`,
//TokenIncludes: `~=`,
//TokenDashMatch: `\|=`,
//TokenPrefixMatch: `\^=`,
//TokenSuffixMatch: `\$=`,
//TokenSubstringMatch: `\*=`,
//TokenChar: `[^"']`,
//TokenBOM: "\uFEFF",
}
// matchers maps the list of tokens to compiled regular expressions.
//
// The map is filled on init() using the macros and productions defined in
// the CSS specification.
var matchers = map[tokenType]*regexp.Regexp{}
// matchOrder is the order to test regexps when first-char shortcuts
// can't be used.
var matchOrder = []tokenType{
TokenURI,
TokenFunction,
TokenUnicodeRange,
TokenIdent,
TokenDimension,
TokenPercentage,
TokenNumber,
TokenCDC,
}
func init() {
// replace macros and compile regexps for productions.
replaceMacro := func(s string) string {
return "(?:" + macros[s[1:len(s)-1]] + ")"
}
for t, s := range productions {
for macroRegexp.MatchString(s) {
s = macroRegexp.ReplaceAllStringFunc(s, replaceMacro)
}
matchers[t] = regexp.MustCompile("^(?:" + s + ")")
}
}
// Scanner --------------------------------------------------------------------
// New returns a new CSS scanner for the given input.
func New(input string) *Scanner {
// Normalize newlines.
input = strings.Replace(input, "\r\n", "\n", -1)
return &Scanner{
input: input,
row: 1,
col: 1,
}
}
// Scanner scans an input and emits tokens following the CSS3 specification.
type Scanner struct {
input string
pos int
row int
col int
err *Token
}
// Next returns the next token from the input.
//
// At the end of the input the token type is TokenEOF.
//
// If the input can't be tokenized the token type is TokenError. This occurs
// in case of unclosed quotation marks or comments.
func (s *Scanner) Next() *Token {
if s.err != nil {
return s.err
}
if s.pos >= len(s.input) {
s.err = &Token{TokenEOF, "", s.row, s.col}
return s.err
}
if s.pos == 0 {
// Test BOM only once, at the beginning of the file.
if strings.HasPrefix(s.input, "\uFEFF") {
return s.emitSimple(TokenBOM, "\uFEFF")
}
}
// There's a lot we can guess based on the first byte so we'll take a
// shortcut before testing multiple regexps.
input := s.input[s.pos:]
switch input[0] {
case '\t', '\n', '\f', '\r', ' ':
// Whitespace.
return s.emitToken(TokenS, matchers[TokenS].FindString(input))
case '.':
// Dot is too common to not have a quick check.
// We'll test if this is a Char; if it is followed by a number it is a
// dimension/percentage/number, and this will be matched later.
if len(input) > 1 && !unicode.IsDigit(rune(input[1])) {
return s.emitSimple(TokenChar, ".")
}
case '#':
// Another common one: Hash or Char.
if match := matchers[TokenHash].FindString(input); match != "" {
return s.emitToken(TokenHash, match)
}
return s.emitSimple(TokenChar, "#")
case '@':
// Another common one: AtKeyword or Char.
if match := matchers[TokenAtKeyword].FindString(input); match != "" {
return s.emitSimple(TokenAtKeyword, match)
}
return s.emitSimple(TokenChar, "@")
case ':', ',', ';', '%', '&', '+', '=', '>', '(', ')', '[', ']', '{', '}':
// More common chars.
return s.emitSimple(TokenChar, string(input[0]))
case '"', '\'':
// String or error.
match := matchers[TokenString].FindString(input)
if match != "" {
return s.emitToken(TokenString, match)
}
s.err = &Token{TokenError, "unclosed quotation mark", s.row, s.col}
return s.err
case '/':
// Comment, error or Char.
if len(input) > 1 && input[1] == '*' {
match := matchers[TokenComment].FindString(input)
if match != "" {
return s.emitToken(TokenComment, match)
} else {
s.err = &Token{TokenError, "unclosed comment", s.row, s.col}
return s.err
}
}
return s.emitSimple(TokenChar, "/")
case '~':
// Includes or Char.
return s.emitPrefixOrChar(TokenIncludes, "~=")
case '|':
// DashMatch or Char.
return s.emitPrefixOrChar(TokenDashMatch, "|=")
case '^':
// PrefixMatch or Char.
return s.emitPrefixOrChar(TokenPrefixMatch, "^=")
case '$':
// SuffixMatch or Char.
return s.emitPrefixOrChar(TokenSuffixMatch, "$=")
case '*':
// SubstringMatch or Char.
return s.emitPrefixOrChar(TokenSubstringMatch, "*=")
case '<':
// CDO or Char.
return s.emitPrefixOrChar(TokenCDO, "<!--")
}
// Test all regexps, in order.
for _, token := range matchOrder {
if match := matchers[token].FindString(input); match != "" {
return s.emitToken(token, match)
}
}
// We already handled unclosed quotation marks and comments,
// so this can only be a Char.
r, width := utf8.DecodeRuneInString(input)
token := &Token{TokenChar, string(r), s.row, s.col}
s.col += width
s.pos += width
return token
}
// updatePosition updates input coordinates based on the consumed text.
func (s *Scanner) updatePosition(text string) {
width := utf8.RuneCountInString(text)
lines := strings.Count(text, "\n")
s.row += lines
if lines == 0 {
s.col += width
} else {
s.col = utf8.RuneCountInString(text[strings.LastIndex(text, "\n"):])
}
s.pos += len(text) // while col is a rune index, pos is a byte index
}
// emitToken returns a Token for the string v and updates the scanner position.
func (s *Scanner) emitToken(t tokenType, v string) *Token {
token := &Token{t, v, s.row, s.col}
s.updatePosition(v)
return token
}
// emitSimple returns a Token for the string v and updates the scanner
// position in a simplified manner.
//
// The string is known to have only ASCII characters and to not have a newline.
func (s *Scanner) emitSimple(t tokenType, v string) *Token {
token := &Token{t, v, s.row, s.col}
s.col += len(v)
s.pos += len(v)
return token
}
// emitPrefixOrChar returns a Token for type t if the current position
// matches the given prefix. Otherwise it returns a Char token using the
// first character from the prefix.
//
// The prefix is known to have only ASCII characters and to not have a newline.
func (s *Scanner) emitPrefixOrChar(t tokenType, prefix string) *Token {
if strings.HasPrefix(s.input[s.pos:], prefix) {
return s.emitSimple(t, prefix)
}
return s.emitSimple(TokenChar, string(prefix[0]))
}

12
vendor/github.com/hashicorp/go-hclog/README.md сгенерированный поставляемый
Просмотреть файл

@@ -17,11 +17,8 @@ JSON output mode for production.
## Stability Note
While this library is fully open source and HashiCorp will be maintaining it
(since we are and will be making extensive use of it), the API and output
format is subject to minor changes as we fully bake and vet it in our projects.
This notice will be removed once it's fully integrated into our major projects
and no further changes are anticipated.
This library has reached 1.0 stability. It's API can be considered solidified
and promised through future versions.
## Installation and Docs
@@ -102,7 +99,7 @@ into all the callers.
### Using `hclog.Fmt()`
```go
var int totalBandwidth = 200
totalBandwidth := 200
appLogger.Info("total bandwidth exceeded", "bandwidth", hclog.Fmt("%d GB/s", totalBandwidth))
```
@@ -146,3 +143,6 @@ log.Printf("[DEBUG] %d", 42)
Notice that if `appLogger` is initialized with the `INFO` log level _and_ you
specify `InferLevels: true`, you will not see any output here. You must change
`appLogger` to `DEBUG` to see output. See the docs for more information.
If the log lines start with a timestamp you can use the
`InferLevelsWithTimestamp` option to try and ignore them.

2
vendor/github.com/hashicorp/go-hclog/global.go сгенерированный поставляемый
Просмотреть файл

@@ -2,6 +2,7 @@ package hclog
import (
"sync"
"time"
)
var (
@@ -14,6 +15,7 @@ var (
DefaultOptions = &LoggerOptions{
Level: DefaultLevel,
Output: DefaultOutput,
TimeFn: time.Now,
}
)

7
vendor/github.com/hashicorp/go-hclog/interceptlogger.go сгенерированный поставляемый
Просмотреть файл

@@ -180,9 +180,10 @@ func (i *interceptLogger) StandardWriterIntercept(opts *StandardLoggerOptions) i
func (i *interceptLogger) StandardWriter(opts *StandardLoggerOptions) io.Writer {
return &stdlogAdapter{
log: i,
inferLevels: opts.InferLevels,
forceLevel: opts.ForceLevel,
log: i,
inferLevels: opts.InferLevels,
inferLevelsWithTimestamp: opts.InferLevelsWithTimestamp,
forceLevel: opts.ForceLevel,
}
}

15
vendor/github.com/hashicorp/go-hclog/intlogger.go сгенерированный поставляемый
Просмотреть файл

@@ -60,6 +60,7 @@ type intLogger struct {
callerOffset int
name string
timeFormat string
timeFn TimeFunction
disableTime bool
// This is an interface so that it's shared by any derived loggers, since
@@ -116,6 +117,7 @@ func newLogger(opts *LoggerOptions) *intLogger {
json: opts.JSONFormat,
name: opts.Name,
timeFormat: TimeFormat,
timeFn: time.Now,
disableTime: opts.DisableTime,
mutex: mutex,
writer: newWriter(output, opts.Color),
@@ -130,6 +132,9 @@ func newLogger(opts *LoggerOptions) *intLogger {
if l.json {
l.timeFormat = TimeFormatJSON
}
if opts.TimeFn != nil {
l.timeFn = opts.TimeFn
}
if opts.TimeFormat != "" {
l.timeFormat = opts.TimeFormat
}
@@ -152,7 +157,7 @@ func (l *intLogger) log(name string, level Level, msg string, args ...interface{
return
}
t := time.Now()
t := l.timeFn()
l.mutex.Lock()
defer l.mutex.Unlock()
@@ -281,6 +286,7 @@ func (l *intLogger) logPlain(t time.Time, name string, level Level, msg string,
val = st
if st == "" {
val = `""`
raw = true
}
case int:
val = strconv.FormatInt(int64(st), 10)
@@ -703,9 +709,10 @@ func (l *intLogger) StandardWriter(opts *StandardLoggerOptions) io.Writer {
newLog.callerOffset = l.callerOffset + 4
}
return &stdlogAdapter{
log: &newLog,
inferLevels: opts.InferLevels,
forceLevel: opts.ForceLevel,
log: &newLog,
inferLevels: opts.InferLevels,
inferLevelsWithTimestamp: opts.InferLevelsWithTimestamp,
forceLevel: opts.ForceLevel,
}
}

15
vendor/github.com/hashicorp/go-hclog/logger.go сгенерированный поставляемый
Просмотреть файл

@@ -5,6 +5,7 @@ import (
"log"
"os"
"strings"
"time"
)
var (
@@ -212,6 +213,15 @@ type StandardLoggerOptions struct {
// [DEBUG] and strip it off before reapplying it.
InferLevels bool
// Indicate that some minimal parsing should be done on strings to try
// and detect their level and re-emit them while ignoring possible
// timestamp values in the beginning of the string.
// This supports the strings like [ERROR], [ERR] [TRACE], [WARN], [INFO],
// [DEBUG] and strip it off before reapplying it.
// The timestamp detection may result in false positives and incomplete
// string outputs.
InferLevelsWithTimestamp bool
// ForceLevel is used to force all output from the standard logger to be at
// the specified level. Similar to InferLevels, this will strip any level
// prefix contained in the logged string before applying the forced level.
@@ -219,6 +229,8 @@ type StandardLoggerOptions struct {
ForceLevel Level
}
type TimeFunction = func() time.Time
// LoggerOptions can be used to configure a new logger.
type LoggerOptions struct {
// Name of the subsystem to prefix logs with
@@ -248,6 +260,9 @@ type LoggerOptions struct {
// The time format to use instead of the default
TimeFormat string
// A function which is called to get the time object that is formatted using `TimeFormat`
TimeFn TimeFunction
// Control whether or not to display the time at all. This is required
// because setting TimeFormat to empty assumes the default format.
DisableTime bool

21
vendor/github.com/hashicorp/go-hclog/stdlog.go сгенерированный поставляемый
Просмотреть файл

@@ -3,16 +3,22 @@ package hclog
import (
"bytes"
"log"
"regexp"
"strings"
)
// Regex to ignore characters commonly found in timestamp formats from the
// beginning of inputs.
var logTimestampRegexp = regexp.MustCompile(`^[\d\s\:\/\.\+-TZ]*`)
// Provides a io.Writer to shim the data out of *log.Logger
// and back into our Logger. This is basically the only way to
// build upon *log.Logger.
type stdlogAdapter struct {
log Logger
inferLevels bool
forceLevel Level
log Logger
inferLevels bool
inferLevelsWithTimestamp bool
forceLevel Level
}
// Take the data, infer the levels if configured, and send it through
@@ -28,6 +34,10 @@ func (s *stdlogAdapter) Write(data []byte) (int, error) {
// Log at the forced level
s.dispatch(str, s.forceLevel)
} else if s.inferLevels {
if s.inferLevelsWithTimestamp {
str = s.trimTimestamp(str)
}
level, str := s.pickLevel(str)
s.dispatch(str, level)
} else {
@@ -74,6 +84,11 @@ func (s *stdlogAdapter) pickLevel(str string) (Level, string) {
}
}
func (s *stdlogAdapter) trimTimestamp(str string) string {
idx := logTimestampRegexp.FindStringIndex(str)
return str[idx[1]:]
}
type logWriter struct {
l *log.Logger
}

7
vendor/github.com/hashicorp/memberlist/net.go сгенерированный поставляемый
Просмотреть файл

@@ -7,6 +7,7 @@ import (
"fmt"
"hash/crc32"
"io"
"math"
"net"
"sync/atomic"
"time"
@@ -1089,6 +1090,12 @@ func (m *Memberlist) decryptRemoteState(bufConn io.Reader, streamLabel string) (
moreBytes := binary.BigEndian.Uint32(cipherText.Bytes()[1:5])
if moreBytes > maxPushStateBytes {
return nil, fmt.Errorf("Remote node state is larger than limit (%d)", moreBytes)
}
//Start reporting the size before you cross the limit
if moreBytes > uint32(math.Floor(.6*maxPushStateBytes)) {
m.logger.Printf("[WARN] memberlist: Remote node state size is (%d) limit is (%d)", moreBytes, maxPushStateBytes)
}
// Read in the rest of the payload

4
vendor/github.com/hashicorp/memberlist/util.go сгенерированный поставляемый
Просмотреть файл

@@ -96,13 +96,13 @@ func pushPullScale(interval time.Duration, n int) time.Duration {
return time.Duration(multiplier) * interval
}
// moveDeadNodes moves nodes that are dead and beyond the gossip to the dead interval
// moveDeadNodes moves dead and left nodes that that have not changed during the gossipToTheDeadTime interval
// to the end of the slice and returns the index of the first moved node.
func moveDeadNodes(nodes []*nodeState, gossipToTheDeadTime time.Duration) int {
numDead := 0
n := len(nodes)
for i := 0; i < n-numDead; i++ {
if nodes[i].State != StateDead {
if !nodes[i].DeadOrLeft() {
continue
}

4
vendor/github.com/klauspost/compress/.goreleaser.yml сгенерированный поставляемый
Просмотреть файл

@@ -3,6 +3,7 @@
before:
hooks:
- ./gen.sh
- go install mvdan.cc/garble@latest
builds:
-
@@ -31,6 +32,7 @@ builds:
- mips64le
goarm:
- 7
gobinary: garble
-
id: "s2d"
binary: s2d
@@ -57,6 +59,7 @@ builds:
- mips64le
goarm:
- 7
gobinary: garble
-
id: "s2sx"
binary: s2sx
@@ -84,6 +87,7 @@ builds:
- mips64le
goarm:
- 7
gobinary: garble
archives:
-

14
vendor/github.com/klauspost/compress/README.md сгенерированный поставляемый
Просмотреть файл

@@ -17,6 +17,13 @@ This package provides various compression algorithms.
# changelog
* Jan 11, 2022 (v1.14.1)
* s2: Add stream index in [#462](https://github.com/klauspost/compress/pull/462)
* flate: Speed and efficiency improvements in [#439](https://github.com/klauspost/compress/pull/439) [#461](https://github.com/klauspost/compress/pull/461) [#455](https://github.com/klauspost/compress/pull/455) [#452](https://github.com/klauspost/compress/pull/452) [#458](https://github.com/klauspost/compress/pull/458)
* zstd: Performance improvement in [#420]( https://github.com/klauspost/compress/pull/420) [#456](https://github.com/klauspost/compress/pull/456) [#437](https://github.com/klauspost/compress/pull/437) [#467](https://github.com/klauspost/compress/pull/467) [#468](https://github.com/klauspost/compress/pull/468)
* zstd: add arm64 xxhash assembly in [#464](https://github.com/klauspost/compress/pull/464)
* Add garbled for binaries for s2 in [#445](https://github.com/klauspost/compress/pull/445)
* Aug 30, 2021 (v1.13.5)
* gz/zlib/flate: Alias stdlib errors [#425](https://github.com/klauspost/compress/pull/425)
* s2: Add block support to commandline tools [#413](https://github.com/klauspost/compress/pull/413)
@@ -432,6 +439,13 @@ For more information see my blog post on [Fast Linear Time Compression](http://b
This is implemented on Go 1.7 as "Huffman Only" mode, though not exposed for gzip.
# Other packages
Here are other packages of good quality and pure Go (no cgo wrappers or autoconverted code):
* [github.com/pierrec/lz4](https://github.com/pierrec/lz4) - strong multithreaded LZ4 compression.
* [github.com/cosnicolaou/pbzip2](https://github.com/cosnicolaou/pbzip2) - multithreaded bzip2 decompression.
* [github.com/dsnet/compress](https://github.com/dsnet/compress) - brotli decompression, bzip2 writer.
# license

167
vendor/github.com/klauspost/compress/flate/deflate.go сгенерированный поставляемый
Просмотреть файл

@@ -6,6 +6,7 @@
package flate
import (
"encoding/binary"
"fmt"
"io"
"math"
@@ -37,15 +38,17 @@ const (
maxMatchLength = 258 // The longest match for the compressor
minOffsetSize = 1 // The shortest offset that makes any sense
// The maximum number of tokens we put into a single flat block, just too
// stop things from getting too large.
maxFlateBlockTokens = 1 << 14
// The maximum number of tokens we will encode at the time.
// Smaller sizes usually creates less optimal blocks.
// Bigger can make context switching slow.
// We use this for levels 7-9, so we make it big.
maxFlateBlockTokens = 1 << 15
maxStoreBlockSize = 65535
hashBits = 17 // After 17 performance degrades
hashSize = 1 << hashBits
hashMask = (1 << hashBits) - 1
hashShift = (hashBits + minMatchLength - 1) / minMatchLength
maxHashOffset = 1 << 24
maxHashOffset = 1 << 28
skipNever = math.MaxInt32
@@ -70,9 +73,9 @@ var levels = []compressionLevel{
{0, 0, 0, 0, 0, 6},
// Levels 7-9 use increasingly more lazy matching
// and increasingly stringent conditions for "good enough".
{8, 8, 24, 16, skipNever, 7},
{10, 16, 24, 64, skipNever, 8},
{32, 258, 258, 4096, skipNever, 9},
{8, 12, 16, 24, skipNever, 7},
{16, 30, 40, 64, skipNever, 8},
{32, 258, 258, 1024, skipNever, 9},
}
// advancedState contains state for the advanced levels, with bigger hash tables, etc.
@@ -93,8 +96,9 @@ type advancedState struct {
hashOffset int
// input window: unprocessed data is window[index:windowEnd]
index int
hashMatch [maxMatchLength + minMatchLength]uint32
index int
estBitsPerByte int
hashMatch [maxMatchLength + minMatchLength]uint32
hash uint32
ii uint16 // position of last match, intended to overflow to reset.
@@ -103,6 +107,7 @@ type advancedState struct {
type compressor struct {
compressionLevel
h *huffmanEncoder
w *huffmanBitWriter
// compression algorithm
@@ -170,7 +175,8 @@ func (d *compressor) writeBlock(tok *tokens, index int, eof bool) error {
window = d.window[d.blockStart:index]
}
d.blockStart = index
d.w.writeBlock(tok, eof, window)
//d.w.writeBlock(tok, eof, window)
d.w.writeBlockDynamic(tok, eof, window, d.sync)
return d.w.err
}
return nil
@@ -263,7 +269,7 @@ func (d *compressor) fillWindow(b []byte) {
// Try to find a match starting at index whose length is greater than prevSize.
// We only look at chainCount possibilities before giving up.
// pos = s.index, prevHead = s.chainHead-s.hashOffset, prevLength=minMatchLength-1, lookahead
func (d *compressor) findMatch(pos int, prevHead int, prevLength int, lookahead int) (length, offset int, ok bool) {
func (d *compressor) findMatch(pos int, prevHead int, lookahead int) (length, offset int, ok bool) {
minMatchLook := maxMatchLength
if lookahead < minMatchLook {
minMatchLook = lookahead
@@ -279,36 +285,75 @@ func (d *compressor) findMatch(pos int, prevHead int, prevLength int, lookahead
// If we've got a match that's good enough, only look in 1/4 the chain.
tries := d.chain
length = prevLength
if length >= d.good {
tries >>= 2
}
length = minMatchLength - 1
wEnd := win[pos+length]
wPos := win[pos:]
minIndex := pos - windowSize
if minIndex < 0 {
minIndex = 0
}
offset = 0
cGain := 0
if d.chain < 100 {
for i := prevHead; tries > 0; tries-- {
if wEnd == win[i+length] {
n := matchLen(win[i:i+minMatchLook], wPos)
if n > length {
length = n
offset = pos - i
ok = true
if n >= nice {
// The match is good enough that we don't try to find a better one.
break
}
wEnd = win[pos+n]
}
}
if i <= minIndex {
// hashPrev[i & windowMask] has already been overwritten, so stop now.
break
}
i = int(d.state.hashPrev[i&windowMask]) - d.state.hashOffset
if i < minIndex {
break
}
}
return
}
// Some like it higher (CSV), some like it lower (JSON)
const baseCost = 6
// Base is 4 bytes at with an additional cost.
// Matches must be better than this.
for i := prevHead; tries > 0; tries-- {
if wEnd == win[i+length] {
n := matchLen(win[i:i+minMatchLook], wPos)
if n > length {
// Calculate gain. Estimate
newGain := d.h.bitLengthRaw(wPos[:n]) - int(offsetExtraBits[offsetCode(uint32(pos-i))]) - baseCost - int(lengthExtraBits[lengthCodes[(n-3)&255]])
if n > length && (n > minMatchLength || pos-i <= 4096) {
length = n
offset = pos - i
ok = true
if n >= nice {
// The match is good enough that we don't try to find a better one.
break
//fmt.Println(n, "gain:", newGain, "prev:", cGain, "raw:", d.h.bitLengthRaw(wPos[:n]))
if newGain > cGain {
length = n
offset = pos - i
cGain = newGain
ok = true
if n >= nice {
// The match is good enough that we don't try to find a better one.
break
}
wEnd = win[pos+n]
}
wEnd = win[pos+n]
}
}
if i == minIndex {
if i <= minIndex {
// hashPrev[i & windowMask] has already been overwritten, so stop now.
break
}
i = int(d.state.hashPrev[i&windowMask]) - d.state.hashOffset
if i < minIndex || i < 0 {
if i < minIndex {
break
}
}
@@ -327,8 +372,7 @@ func (d *compressor) writeStoredBlock(buf []byte) error {
// of the supplied slice.
// The caller must ensure that len(b) >= 4.
func hash4(b []byte) uint32 {
b = b[:4]
return hash4u(uint32(b[3])|uint32(b[2])<<8|uint32(b[1])<<16|uint32(b[0])<<24, hashBits)
return hash4u(binary.LittleEndian.Uint32(b), hashBits)
}
// bulkHash4 will compute hashes using the same
@@ -337,11 +381,12 @@ func bulkHash4(b []byte, dst []uint32) {
if len(b) < 4 {
return
}
hb := uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24
hb := binary.LittleEndian.Uint32(b)
dst[0] = hash4u(hb, hashBits)
end := len(b) - 4 + 1
for i := 1; i < end; i++ {
hb = (hb << 8) | uint32(b[i+3])
hb = (hb >> 8) | uint32(b[i+3])<<24
dst[i] = hash4u(hb, hashBits)
}
}
@@ -374,10 +419,21 @@ func (d *compressor) deflateLazy() {
if d.windowEnd-s.index < minMatchLength+maxMatchLength && !d.sync {
return
}
if d.windowEnd != s.index && d.chain > 100 {
// Get literal huffman coder.
if d.h == nil {
d.h = newHuffmanEncoder(maxFlateBlockTokens)
}
var tmp [256]uint16
for _, v := range d.window[s.index:d.windowEnd] {
tmp[v]++
}
d.h.generate(tmp[:], 15)
}
s.maxInsertIndex = d.windowEnd - (minMatchLength - 1)
if s.index < s.maxInsertIndex {
s.hash = hash4(d.window[s.index : s.index+minMatchLength])
s.hash = hash4(d.window[s.index:])
}
for {
@@ -410,7 +466,7 @@ func (d *compressor) deflateLazy() {
}
if s.index < s.maxInsertIndex {
// Update the hash
s.hash = hash4(d.window[s.index : s.index+minMatchLength])
s.hash = hash4(d.window[s.index:])
ch := s.hashHead[s.hash&hashMask]
s.chainHead = int(ch)
s.hashPrev[s.index&windowMask] = ch
@@ -426,12 +482,37 @@ func (d *compressor) deflateLazy() {
}
if s.chainHead-s.hashOffset >= minIndex && lookahead > prevLength && prevLength < d.lazy {
if newLength, newOffset, ok := d.findMatch(s.index, s.chainHead-s.hashOffset, minMatchLength-1, lookahead); ok {
if newLength, newOffset, ok := d.findMatch(s.index, s.chainHead-s.hashOffset, lookahead); ok {
s.length = newLength
s.offset = newOffset
}
}
if prevLength >= minMatchLength && s.length <= prevLength {
// Check for better match at end...
//
// checkOff must be >=2 since we otherwise risk checking s.index
// Offset of 2 seems to yield best results.
const checkOff = 2
prevIndex := s.index - 1
if prevIndex+prevLength+checkOff < s.maxInsertIndex {
end := lookahead
if lookahead > maxMatchLength {
end = maxMatchLength
}
end += prevIndex
idx := prevIndex + prevLength - (4 - checkOff)
h := hash4(d.window[idx:])
ch2 := int(s.hashHead[h&hashMask]) - s.hashOffset - prevLength + (4 - checkOff)
if ch2 > minIndex {
length := matchLen(d.window[prevIndex:end], d.window[ch2:])
// It seems like a pure length metric is best.
if length > prevLength {
prevLength = length
prevOffset = prevIndex - ch2
}
}
}
// There was a match at the previous step, and the current match is
// not better. Output the previous match.
d.tokens.AddMatch(uint32(prevLength-3), uint32(prevOffset-minOffsetSize))
@@ -479,6 +560,7 @@ func (d *compressor) deflateLazy() {
}
d.tokens.Reset()
}
s.ii = 0
} else {
// Reset, if we got a match this run.
if s.length >= minMatchLength {
@@ -498,13 +580,12 @@ func (d *compressor) deflateLazy() {
// If we have a long run of no matches, skip additional bytes
// Resets when s.ii overflows after 64KB.
if s.ii > 31 {
n := int(s.ii >> 5)
if n := int(s.ii) - d.chain; n > 0 {
n = 1 + int(n>>6)
for j := 0; j < n; j++ {
if s.index >= d.windowEnd-1 {
break
}
d.tokens.AddLiteral(d.window[s.index-1])
if d.tokens.n == maxFlateBlockTokens {
if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil {
@@ -512,6 +593,14 @@ func (d *compressor) deflateLazy() {
}
d.tokens.Reset()
}
// Index...
if s.index < s.maxInsertIndex {
h := hash4(d.window[s.index:])
ch := s.hashHead[h]
s.chainHead = int(ch)
s.hashPrev[s.index&windowMask] = ch
s.hashHead[h] = uint32(s.index + s.hashOffset)
}
s.index++
}
// Flush last byte
@@ -611,7 +700,9 @@ func (d *compressor) write(b []byte) (n int, err error) {
}
n = len(b)
for len(b) > 0 {
d.step(d)
if d.windowEnd == len(d.window) || d.sync {
d.step(d)
}
b = b[d.fill(d, b):]
if d.err != nil {
return 0, d.err
@@ -652,13 +743,13 @@ func (d *compressor) init(w io.Writer, level int) (err error) {
level = 5
fallthrough
case level >= 1 && level <= 6:
d.w.logNewTablePenalty = 8
d.w.logNewTablePenalty = 7
d.fast = newFastEnc(level)
d.window = make([]byte, maxStoreBlockSize)
d.fill = (*compressor).fillBlock
d.step = (*compressor).storeFast
case 7 <= level && level <= 9:
d.w.logNewTablePenalty = 10
d.w.logNewTablePenalty = 8
d.state = &advancedState{}
d.compressionLevel = levels[level]
d.initDeflate()

2
vendor/github.com/klauspost/compress/flate/fast_encoder.go сгенерированный поставляемый
Просмотреть файл

@@ -213,11 +213,9 @@ func (e *fastGen) Reset() {
// matchLen returns the maximum length.
// 'a' must be the shortest of the two.
func matchLen(a, b []byte) int {
b = b[:len(a)]
var checked int
for len(a) >= 8 {
b = b[:len(a)]
if diff := binary.LittleEndian.Uint64(a) ^ binary.LittleEndian.Uint64(b); diff != 0 {
return checked + (bits.TrailingZeros64(diff) >> 3)
}

199
vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go сгенерированный поставляемый
Просмотреть файл

@@ -52,18 +52,18 @@ var lengthBase = [32]uint8{
}
// offset code word extra bits.
var offsetExtraBits = [64]int8{
var offsetExtraBits = [32]int8{
0, 0, 0, 0, 1, 1, 2, 2, 3, 3,
4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
9, 9, 10, 10, 11, 11, 12, 12, 13, 13,
/* extended window */
14, 14, 15, 15, 16, 16, 17, 17, 18, 18, 19, 19, 20, 20,
14, 14,
}
var offsetCombined = [32]uint32{}
func init() {
var offsetBase = [64]uint32{
var offsetBase = [32]uint32{
/* normal deflate */
0x000000, 0x000001, 0x000002, 0x000003, 0x000004,
0x000006, 0x000008, 0x00000c, 0x000010, 0x000018,
@@ -73,9 +73,7 @@ func init() {
0x001800, 0x002000, 0x003000, 0x004000, 0x006000,
/* extended window */
0x008000, 0x00c000, 0x010000, 0x018000, 0x020000,
0x030000, 0x040000, 0x060000, 0x080000, 0x0c0000,
0x100000, 0x180000, 0x200000, 0x300000,
0x008000, 0x00c000,
}
for i := range offsetCombined[:] {
@@ -155,37 +153,33 @@ func (w *huffmanBitWriter) reset(writer io.Writer) {
w.lastHuffMan = false
}
func (w *huffmanBitWriter) canReuse(t *tokens) (offsets, lits bool) {
offsets, lits = true, true
func (w *huffmanBitWriter) canReuse(t *tokens) (ok bool) {
a := t.offHist[:offsetCodeCount]
b := w.offsetFreq[:len(a)]
for i := range a {
if b[i] == 0 && a[i] != 0 {
offsets = false
break
b := w.offsetEncoding.codes
b = b[:len(a)]
for i, v := range a {
if v != 0 && b[i].len == 0 {
return false
}
}
a = t.extraHist[:literalCount-256]
b = w.literalFreq[256:literalCount]
b = w.literalEncoding.codes[256:literalCount]
b = b[:len(a)]
for i := range a {
if b[i] == 0 && a[i] != 0 {
lits = false
break
for i, v := range a {
if v != 0 && b[i].len == 0 {
return false
}
}
if lits {
a = t.litHist[:]
b = w.literalFreq[:len(a)]
for i := range a {
if b[i] == 0 && a[i] != 0 {
lits = false
break
}
a = t.litHist[:256]
b = w.literalEncoding.codes[:len(a)]
for i, v := range a {
if v != 0 && b[i].len == 0 {
return false
}
}
return
return true
}
func (w *huffmanBitWriter) flush() {
@@ -222,7 +216,7 @@ func (w *huffmanBitWriter) write(b []byte) {
}
func (w *huffmanBitWriter) writeBits(b int32, nb uint16) {
w.bits |= uint64(b) << w.nbits
w.bits |= uint64(b) << (w.nbits & 63)
w.nbits += nb
if w.nbits >= 48 {
w.writeOutBits()
@@ -423,7 +417,7 @@ func (w *huffmanBitWriter) storedSize(in []byte) (int, bool) {
func (w *huffmanBitWriter) writeCode(c hcode) {
// The function does not get inlined if we "& 63" the shift.
w.bits |= uint64(c.code) << w.nbits
w.bits |= uint64(c.code) << (w.nbits & 63)
w.nbits += c.len
if w.nbits >= 48 {
w.writeOutBits()
@@ -566,7 +560,7 @@ func (w *huffmanBitWriter) writeBlock(tokens *tokens, eof bool, input []byte) {
w.lastHeader = 0
}
numLiterals, numOffsets := w.indexTokens(tokens, false)
w.generate(tokens)
w.generate()
var extraBits int
storedSize, storable := w.storedSize(input)
if storable {
@@ -595,7 +589,7 @@ func (w *huffmanBitWriter) writeBlock(tokens *tokens, eof bool, input []byte) {
}
// Stored bytes?
if storable && storedSize < size {
if storable && storedSize <= size {
w.writeStoredHeader(len(input), eof)
w.writeBytes(input)
return
@@ -634,22 +628,39 @@ func (w *huffmanBitWriter) writeBlockDynamic(tokens *tokens, eof bool, input []b
w.lastHeader = 0
w.lastHuffMan = false
}
if !sync {
tokens.Fill()
// fillReuse enables filling of empty values.
// This will make encodings always reusable without testing.
// However, this does not appear to benefit on most cases.
const fillReuse = false
// Check if we can reuse...
if !fillReuse && w.lastHeader > 0 && !w.canReuse(tokens) {
w.writeCode(w.literalEncoding.codes[endBlockMarker])
w.lastHeader = 0
}
numLiterals, numOffsets := w.indexTokens(tokens, !sync)
extraBits := 0
ssize, storable := w.storedSize(input)
const usePrefs = true
if storable || w.lastHeader > 0 {
extraBits = w.extraBitSize()
}
var size int
// Check if we should reuse.
if w.lastHeader > 0 {
// Estimate size for using a new table.
// Use the previous header size as the best estimate.
newSize := w.lastHeader + tokens.EstimatedBits()
newSize += newSize >> w.logNewTablePenalty
newSize += int(w.literalEncoding.codes[endBlockMarker].len) + newSize>>w.logNewTablePenalty
// The estimated size is calculated as an optimal table.
// We add a penalty to make it more realistic and re-use a bit more.
reuseSize := w.dynamicReuseSize(w.literalEncoding, w.offsetEncoding) + w.extraBitSize()
reuseSize := w.dynamicReuseSize(w.literalEncoding, w.offsetEncoding) + extraBits
// Check if a new table is better.
if newSize < reuseSize {
@@ -660,35 +671,79 @@ func (w *huffmanBitWriter) writeBlockDynamic(tokens *tokens, eof bool, input []b
} else {
size = reuseSize
}
if preSize := w.fixedSize(extraBits) + 7; usePrefs && preSize < size {
// Check if we get a reasonable size decrease.
if storable && ssize <= size {
w.writeStoredHeader(len(input), eof)
w.writeBytes(input)
return
}
w.writeFixedHeader(eof)
if !sync {
tokens.AddEOB()
}
w.writeTokens(tokens.Slice(), fixedLiteralEncoding.codes, fixedOffsetEncoding.codes)
return
}
// Check if we get a reasonable size decrease.
if ssize, storable := w.storedSize(input); storable && ssize < (size+size>>4) {
if storable && ssize <= size {
w.writeStoredHeader(len(input), eof)
w.writeBytes(input)
w.lastHeader = 0
return
}
}
// We want a new block/table
if w.lastHeader == 0 {
w.generate(tokens)
if fillReuse && !sync {
w.fillTokens()
numLiterals, numOffsets = maxNumLit, maxNumDist
} else {
w.literalFreq[endBlockMarker] = 1
}
w.generate()
// Generate codegen and codegenFrequencies, which indicates how to encode
// the literalEncoding and the offsetEncoding.
w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, w.offsetEncoding)
w.codegenEncoding.generate(w.codegenFreq[:], 7)
var numCodegens int
size, numCodegens = w.dynamicSize(w.literalEncoding, w.offsetEncoding, w.extraBitSize())
// Store bytes, if we don't get a reasonable improvement.
if ssize, storable := w.storedSize(input); storable && ssize < (size+size>>4) {
if fillReuse && !sync {
// Reindex for accurate size...
w.indexTokens(tokens, true)
}
size, numCodegens = w.dynamicSize(w.literalEncoding, w.offsetEncoding, extraBits)
// Store predefined, if we don't get a reasonable improvement.
if preSize := w.fixedSize(extraBits); usePrefs && preSize <= size {
// Store bytes, if we don't get an improvement.
if storable && ssize <= preSize {
w.writeStoredHeader(len(input), eof)
w.writeBytes(input)
return
}
w.writeFixedHeader(eof)
if !sync {
tokens.AddEOB()
}
w.writeTokens(tokens.Slice(), fixedLiteralEncoding.codes, fixedOffsetEncoding.codes)
return
}
if storable && ssize <= size {
// Store bytes, if we don't get an improvement.
w.writeStoredHeader(len(input), eof)
w.writeBytes(input)
w.lastHeader = 0
return
}
// Write Huffman table.
w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof)
w.lastHeader, _ = w.headerSize()
if !sync {
w.lastHeader, _ = w.headerSize()
}
w.lastHuffMan = false
}
@@ -699,6 +754,19 @@ func (w *huffmanBitWriter) writeBlockDynamic(tokens *tokens, eof bool, input []b
w.writeTokens(tokens.Slice(), w.literalEncoding.codes, w.offsetEncoding.codes)
}
func (w *huffmanBitWriter) fillTokens() {
for i, v := range w.literalFreq[:literalCount] {
if v == 0 {
w.literalFreq[i] = 1
}
}
for i, v := range w.offsetFreq[:offsetCodeCount] {
if v == 0 {
w.offsetFreq[i] = 1
}
}
}
// indexTokens indexes a slice of tokens, and updates
// literalFreq and offsetFreq, and generates literalEncoding
// and offsetEncoding.
@@ -733,7 +801,7 @@ func (w *huffmanBitWriter) indexTokens(t *tokens, filled bool) (numLiterals, num
return
}
func (w *huffmanBitWriter) generate(t *tokens) {
func (w *huffmanBitWriter) generate() {
w.literalEncoding.generate(w.literalFreq[:literalCount], 15)
w.offsetEncoding.generate(w.offsetFreq[:offsetCodeCount], 15)
}
@@ -768,7 +836,7 @@ func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode)
if t < matchType {
//w.writeCode(lits[t.literal()])
c := lits[t.literal()]
bits |= uint64(c.code) << nbits
bits |= uint64(c.code) << (nbits & 63)
nbits += c.len
if nbits >= 48 {
binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits)
@@ -796,7 +864,7 @@ func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode)
} else {
// inlined
c := lengths[lengthCode&31]
bits |= uint64(c.code) << nbits
bits |= uint64(c.code) << (nbits & 63)
nbits += c.len
if nbits >= 48 {
binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits)
@@ -819,7 +887,7 @@ func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode)
if extraLengthBits > 0 {
//w.writeBits(extraLength, extraLengthBits)
extraLength := int32(length - lengthBase[lengthCode&31])
bits |= uint64(extraLength) << nbits
bits |= uint64(extraLength) << (nbits & 63)
nbits += extraLengthBits
if nbits >= 48 {
binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits)
@@ -846,7 +914,7 @@ func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode)
} else {
// inlined
c := offs[offsetCode]
bits |= uint64(c.code) << nbits
bits |= uint64(c.code) << (nbits & 63)
nbits += c.len
if nbits >= 48 {
binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits)
@@ -867,7 +935,7 @@ func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode)
offsetComb := offsetCombined[offsetCode]
if offsetComb > 1<<16 {
//w.writeBits(extraOffset, extraOffsetBits)
bits |= uint64(offset&matchOffsetOnlyMask-(offsetComb&0xffff)) << nbits
bits |= uint64(offset-(offsetComb&0xffff)) << (nbits & 63)
nbits += uint16(offsetComb >> 16)
if nbits >= 48 {
binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits)
@@ -996,10 +1064,41 @@ func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte, sync bool) {
encoding := w.literalEncoding.codes[:256]
// Go 1.16 LOVES having these on stack. At least 1.5x the speed.
bits, nbits, nbytes := w.bits, w.nbits, w.nbytes
// Unroll, write 3 codes/loop.
// Fastest number of unrolls.
for len(input) > 3 {
// We must have at least 48 bits free.
if nbits >= 8 {
n := nbits >> 3
binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits)
bits >>= (n * 8) & 63
nbits -= n * 8
nbytes += uint8(n)
}
if nbytes >= bufferFlushSize {
if w.err != nil {
nbytes = 0
return
}
_, w.err = w.writer.Write(w.bytes[:nbytes])
nbytes = 0
}
a, b := encoding[input[0]], encoding[input[1]]
bits |= uint64(a.code) << (nbits & 63)
bits |= uint64(b.code) << ((nbits + a.len) & 63)
c := encoding[input[2]]
nbits += b.len + a.len
bits |= uint64(c.code) << (nbits & 63)
nbits += c.len
input = input[3:]
}
// Remaining...
for _, t := range input {
// Bitwriting inlined, ~30% speedup
c := encoding[t]
bits |= uint64(c.code) << nbits
bits |= uint64(c.code) << (nbits & 63)
nbits += c.len
if debugDeflate {
count += int(c.len)

4
vendor/github.com/klauspost/compress/flate/huffman_code.go сгенерированный поставляемый
Просмотреть файл

@@ -129,9 +129,7 @@ func (h *huffmanEncoder) bitLength(freq []uint16) int {
func (h *huffmanEncoder) bitLengthRaw(b []byte) int {
var total int
for _, f := range b {
if f != 0 {
total += int(h.codes[f].len)
}
total += int(h.codes[f].len)
}
return total
}

9
vendor/github.com/klauspost/compress/flate/inflate.go сгенерированный поставляемый
Просмотреть файл

@@ -328,11 +328,17 @@ func (f *decompressor) nextBlock() {
switch typ {
case 0:
f.dataBlock()
if debugDecode {
fmt.Println("stored block")
}
case 1:
// compressed, fixed Huffman tables
f.hl = &fixedHuffmanDecoder
f.hd = nil
f.huffmanBlockDecoder()()
if debugDecode {
fmt.Println("predefinied huffman block")
}
case 2:
// compressed, dynamic Huffman tables
if f.err = f.readHuffman(); f.err != nil {
@@ -341,6 +347,9 @@ func (f *decompressor) nextBlock() {
f.hl = &f.h1
f.hd = &f.h2
f.huffmanBlockDecoder()()
if debugDecode {
fmt.Println("dynamic huffman block")
}
default:
// 3 is reserved.
if debugDecode {

19
vendor/github.com/klauspost/compress/flate/token.go сгенерированный поставляемый
Просмотреть файл

@@ -129,11 +129,11 @@ var offsetCodes14 = [256]uint32{
type token uint32
type tokens struct {
nLits int
extraHist [32]uint16 // codes 256->maxnumlit
offHist [32]uint16 // offset codes
litHist [256]uint16 // codes 0->255
n uint16 // Must be able to contain maxStoreBlockSize
nFilled int
n uint16 // Must be able to contain maxStoreBlockSize
tokens [maxStoreBlockSize + 1]token
}
@@ -142,7 +142,7 @@ func (t *tokens) Reset() {
return
}
t.n = 0
t.nLits = 0
t.nFilled = 0
for i := range t.litHist[:] {
t.litHist[i] = 0
}
@@ -161,12 +161,12 @@ func (t *tokens) Fill() {
for i, v := range t.litHist[:] {
if v == 0 {
t.litHist[i] = 1
t.nLits++
t.nFilled++
}
}
for i, v := range t.extraHist[:literalCount-256] {
if v == 0 {
t.nLits++
t.nFilled++
t.extraHist[i] = 1
}
}
@@ -202,14 +202,12 @@ func emitLiteral(dst *tokens, lit []byte) {
dst.litHist[v]++
}
dst.n += uint16(len(lit))
dst.nLits += len(lit)
}
func (t *tokens) AddLiteral(lit byte) {
t.tokens[t.n] = token(lit)
t.litHist[lit]++
t.n++
t.nLits++
}
// from https://stackoverflow.com/a/28730362
@@ -230,8 +228,9 @@ func (t *tokens) EstimatedBits() int {
shannon := float32(0)
bits := int(0)
nMatches := 0
if t.nLits > 0 {
invTotal := 1.0 / float32(t.nLits)
total := int(t.n) + t.nFilled
if total > 0 {
invTotal := 1.0 / float32(total)
for _, v := range t.litHist[:] {
if v > 0 {
n := float32(v)
@@ -275,7 +274,6 @@ func (t *tokens) AddMatch(xlength uint32, xoffset uint32) {
}
oCode := offsetCode(xoffset)
xoffset |= oCode << 16
t.nLits++
t.extraHist[lengthCodes1[uint8(xlength)]]++
t.offHist[oCode]++
@@ -301,7 +299,6 @@ func (t *tokens) AddMatchLong(xlength int32, xoffset uint32) {
}
xlength -= xl
xl -= baseMatchLength
t.nLits++
t.extraHist[lengthCodes1[uint8(xl)]]++
t.offHist[oc]++
t.tokens[t.n] = token(matchType | uint32(xl)<<lengthShift | xoffset)

230
vendor/github.com/klauspost/compress/huff0/decompress.go сгенерированный поставляемый
Просмотреть файл

@@ -20,7 +20,7 @@ type dEntrySingle struct {
// double-symbols decoding
type dEntryDouble struct {
seq uint16
seq [4]byte
nBits uint8
len uint8
}
@@ -753,23 +753,21 @@ func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) {
br[stream2].fillFast()
val := br[stream].peekBitsFast(d.actualTableLog)
v := single[val&tlMask]
br[stream].advance(uint8(v.entry))
buf[off+bufoff*stream] = uint8(v.entry >> 8)
val2 := br[stream2].peekBitsFast(d.actualTableLog)
v := single[val&tlMask]
v2 := single[val2&tlMask]
br[stream].advance(uint8(v.entry))
br[stream2].advance(uint8(v2.entry))
buf[off+bufoff*stream] = uint8(v.entry >> 8)
buf[off+bufoff*stream2] = uint8(v2.entry >> 8)
val = br[stream].peekBitsFast(d.actualTableLog)
v = single[val&tlMask]
br[stream].advance(uint8(v.entry))
buf[off+bufoff*stream+1] = uint8(v.entry >> 8)
val2 = br[stream2].peekBitsFast(d.actualTableLog)
v = single[val&tlMask]
v2 = single[val2&tlMask]
br[stream].advance(uint8(v.entry))
br[stream2].advance(uint8(v2.entry))
buf[off+bufoff*stream+1] = uint8(v.entry >> 8)
buf[off+bufoff*stream2+1] = uint8(v2.entry >> 8)
}
@@ -780,23 +778,21 @@ func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) {
br[stream2].fillFast()
val := br[stream].peekBitsFast(d.actualTableLog)
v := single[val&tlMask]
br[stream].advance(uint8(v.entry))
buf[off+bufoff*stream] = uint8(v.entry >> 8)
val2 := br[stream2].peekBitsFast(d.actualTableLog)
v := single[val&tlMask]
v2 := single[val2&tlMask]
br[stream].advance(uint8(v.entry))
br[stream2].advance(uint8(v2.entry))
buf[off+bufoff*stream] = uint8(v.entry >> 8)
buf[off+bufoff*stream2] = uint8(v2.entry >> 8)
val = br[stream].peekBitsFast(d.actualTableLog)
v = single[val&tlMask]
br[stream].advance(uint8(v.entry))
buf[off+bufoff*stream+1] = uint8(v.entry >> 8)
val2 = br[stream2].peekBitsFast(d.actualTableLog)
v = single[val&tlMask]
v2 = single[val2&tlMask]
br[stream].advance(uint8(v.entry))
br[stream2].advance(uint8(v2.entry))
buf[off+bufoff*stream+1] = uint8(v.entry >> 8)
buf[off+bufoff*stream2+1] = uint8(v2.entry >> 8)
}
@@ -914,7 +910,7 @@ func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) {
out := dst
dstEvery := (dstSize + 3) / 4
shift := (8 - d.actualTableLog) & 7
shift := (56 + (8 - d.actualTableLog)) & 63
const tlSize = 1 << 8
single := d.dt.single[:tlSize]
@@ -935,79 +931,91 @@ func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) {
// Interleave 2 decodes.
const stream = 0
const stream2 = 1
br[stream].fillFast()
br[stream2].fillFast()
br1 := &br[stream]
br2 := &br[stream2]
br1.fillFast()
br2.fillFast()
v := single[br[stream].peekByteFast()>>shift].entry
v := single[uint8(br1.value>>shift)].entry
v2 := single[uint8(br2.value>>shift)].entry
br1.bitsRead += uint8(v)
br1.value <<= v & 63
br2.bitsRead += uint8(v2)
br2.value <<= v2 & 63
buf[off+bufoff*stream] = uint8(v >> 8)
br[stream].advance(uint8(v))
v2 := single[br[stream2].peekByteFast()>>shift].entry
buf[off+bufoff*stream2] = uint8(v2 >> 8)
br[stream2].advance(uint8(v2))
v = single[br[stream].peekByteFast()>>shift].entry
v = single[uint8(br1.value>>shift)].entry
v2 = single[uint8(br2.value>>shift)].entry
br1.bitsRead += uint8(v)
br1.value <<= v & 63
br2.bitsRead += uint8(v2)
br2.value <<= v2 & 63
buf[off+bufoff*stream+1] = uint8(v >> 8)
br[stream].advance(uint8(v))
v2 = single[br[stream2].peekByteFast()>>shift].entry
buf[off+bufoff*stream2+1] = uint8(v2 >> 8)
br[stream2].advance(uint8(v2))
v = single[br[stream].peekByteFast()>>shift].entry
v = single[uint8(br1.value>>shift)].entry
v2 = single[uint8(br2.value>>shift)].entry
br1.bitsRead += uint8(v)
br1.value <<= v & 63
br2.bitsRead += uint8(v2)
br2.value <<= v2 & 63
buf[off+bufoff*stream+2] = uint8(v >> 8)
br[stream].advance(uint8(v))
v2 = single[br[stream2].peekByteFast()>>shift].entry
buf[off+bufoff*stream2+2] = uint8(v2 >> 8)
br[stream2].advance(uint8(v2))
v = single[br[stream].peekByteFast()>>shift].entry
buf[off+bufoff*stream+3] = uint8(v >> 8)
br[stream].advance(uint8(v))
v2 = single[br[stream2].peekByteFast()>>shift].entry
v = single[uint8(br1.value>>shift)].entry
v2 = single[uint8(br2.value>>shift)].entry
br1.bitsRead += uint8(v)
br1.value <<= v & 63
br2.bitsRead += uint8(v2)
br2.value <<= v2 & 63
buf[off+bufoff*stream2+3] = uint8(v2 >> 8)
br[stream2].advance(uint8(v2))
buf[off+bufoff*stream+3] = uint8(v >> 8)
}
{
const stream = 2
const stream2 = 3
br[stream].fillFast()
br[stream2].fillFast()
br1 := &br[stream]
br2 := &br[stream2]
br1.fillFast()
br2.fillFast()
v := single[br[stream].peekByteFast()>>shift].entry
v := single[uint8(br1.value>>shift)].entry
v2 := single[uint8(br2.value>>shift)].entry
br1.bitsRead += uint8(v)
br1.value <<= v & 63
br2.bitsRead += uint8(v2)
br2.value <<= v2 & 63
buf[off+bufoff*stream] = uint8(v >> 8)
br[stream].advance(uint8(v))
v2 := single[br[stream2].peekByteFast()>>shift].entry
buf[off+bufoff*stream2] = uint8(v2 >> 8)
br[stream2].advance(uint8(v2))
v = single[br[stream].peekByteFast()>>shift].entry
v = single[uint8(br1.value>>shift)].entry
v2 = single[uint8(br2.value>>shift)].entry
br1.bitsRead += uint8(v)
br1.value <<= v & 63
br2.bitsRead += uint8(v2)
br2.value <<= v2 & 63
buf[off+bufoff*stream+1] = uint8(v >> 8)
br[stream].advance(uint8(v))
v2 = single[br[stream2].peekByteFast()>>shift].entry
buf[off+bufoff*stream2+1] = uint8(v2 >> 8)
br[stream2].advance(uint8(v2))
v = single[br[stream].peekByteFast()>>shift].entry
v = single[uint8(br1.value>>shift)].entry
v2 = single[uint8(br2.value>>shift)].entry
br1.bitsRead += uint8(v)
br1.value <<= v & 63
br2.bitsRead += uint8(v2)
br2.value <<= v2 & 63
buf[off+bufoff*stream+2] = uint8(v >> 8)
br[stream].advance(uint8(v))
v2 = single[br[stream2].peekByteFast()>>shift].entry
buf[off+bufoff*stream2+2] = uint8(v2 >> 8)
br[stream2].advance(uint8(v2))
v = single[br[stream].peekByteFast()>>shift].entry
buf[off+bufoff*stream+3] = uint8(v >> 8)
br[stream].advance(uint8(v))
v2 = single[br[stream2].peekByteFast()>>shift].entry
v = single[uint8(br1.value>>shift)].entry
v2 = single[uint8(br2.value>>shift)].entry
br1.bitsRead += uint8(v)
br1.value <<= v & 63
br2.bitsRead += uint8(v2)
br2.value <<= v2 & 63
buf[off+bufoff*stream2+3] = uint8(v2 >> 8)
br[stream2].advance(uint8(v2))
buf[off+bufoff*stream+3] = uint8(v >> 8)
}
off += 4
@@ -1073,7 +1081,7 @@ func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) {
}
// Read value and increment offset.
v := single[br.peekByteFast()>>shift].entry
v := single[uint8(br.value>>shift)].entry
nBits := uint8(v)
br.advance(nBits)
bitsLeft -= int(nBits)
@@ -1121,7 +1129,7 @@ func (d *Decoder) decompress4X8bitExactly(dst, src []byte) ([]byte, error) {
out := dst
dstEvery := (dstSize + 3) / 4
const shift = 0
const shift = 56
const tlSize = 1 << 8
const tlMask = tlSize - 1
single := d.dt.single[:tlSize]
@@ -1145,37 +1153,41 @@ func (d *Decoder) decompress4X8bitExactly(dst, src []byte) ([]byte, error) {
br[stream].fillFast()
br[stream2].fillFast()
v := single[br[stream].peekByteFast()>>shift].entry
v := single[uint8(br[stream].value>>shift)].entry
v2 := single[uint8(br[stream2].value>>shift)].entry
br[stream].bitsRead += uint8(v)
br[stream].value <<= v & 63
br[stream2].bitsRead += uint8(v2)
br[stream2].value <<= v2 & 63
buf[off+bufoff*stream] = uint8(v >> 8)
br[stream].advance(uint8(v))
v2 := single[br[stream2].peekByteFast()>>shift].entry
buf[off+bufoff*stream2] = uint8(v2 >> 8)
br[stream2].advance(uint8(v2))
v = single[br[stream].peekByteFast()>>shift].entry
v = single[uint8(br[stream].value>>shift)].entry
v2 = single[uint8(br[stream2].value>>shift)].entry
br[stream].bitsRead += uint8(v)
br[stream].value <<= v & 63
br[stream2].bitsRead += uint8(v2)
br[stream2].value <<= v2 & 63
buf[off+bufoff*stream+1] = uint8(v >> 8)
br[stream].advance(uint8(v))
v2 = single[br[stream2].peekByteFast()>>shift].entry
buf[off+bufoff*stream2+1] = uint8(v2 >> 8)
br[stream2].advance(uint8(v2))
v = single[br[stream].peekByteFast()>>shift].entry
v = single[uint8(br[stream].value>>shift)].entry
v2 = single[uint8(br[stream2].value>>shift)].entry
br[stream].bitsRead += uint8(v)
br[stream].value <<= v & 63
br[stream2].bitsRead += uint8(v2)
br[stream2].value <<= v2 & 63
buf[off+bufoff*stream+2] = uint8(v >> 8)
br[stream].advance(uint8(v))
v2 = single[br[stream2].peekByteFast()>>shift].entry
buf[off+bufoff*stream2+2] = uint8(v2 >> 8)
br[stream2].advance(uint8(v2))
v = single[br[stream].peekByteFast()>>shift].entry
v = single[uint8(br[stream].value>>shift)].entry
v2 = single[uint8(br[stream2].value>>shift)].entry
br[stream].bitsRead += uint8(v)
br[stream].value <<= v & 63
br[stream2].bitsRead += uint8(v2)
br[stream2].value <<= v2 & 63
buf[off+bufoff*stream+3] = uint8(v >> 8)
br[stream].advance(uint8(v))
v2 = single[br[stream2].peekByteFast()>>shift].entry
buf[off+bufoff*stream2+3] = uint8(v2 >> 8)
br[stream2].advance(uint8(v2))
}
{
@@ -1184,37 +1196,41 @@ func (d *Decoder) decompress4X8bitExactly(dst, src []byte) ([]byte, error) {
br[stream].fillFast()
br[stream2].fillFast()
v := single[br[stream].peekByteFast()>>shift].entry
v := single[uint8(br[stream].value>>shift)].entry
v2 := single[uint8(br[stream2].value>>shift)].entry
br[stream].bitsRead += uint8(v)
br[stream].value <<= v & 63
br[stream2].bitsRead += uint8(v2)
br[stream2].value <<= v2 & 63
buf[off+bufoff*stream] = uint8(v >> 8)
br[stream].advance(uint8(v))
v2 := single[br[stream2].peekByteFast()>>shift].entry
buf[off+bufoff*stream2] = uint8(v2 >> 8)
br[stream2].advance(uint8(v2))
v = single[br[stream].peekByteFast()>>shift].entry
v = single[uint8(br[stream].value>>shift)].entry
v2 = single[uint8(br[stream2].value>>shift)].entry
br[stream].bitsRead += uint8(v)
br[stream].value <<= v & 63
br[stream2].bitsRead += uint8(v2)
br[stream2].value <<= v2 & 63
buf[off+bufoff*stream+1] = uint8(v >> 8)
br[stream].advance(uint8(v))
v2 = single[br[stream2].peekByteFast()>>shift].entry
buf[off+bufoff*stream2+1] = uint8(v2 >> 8)
br[stream2].advance(uint8(v2))
v = single[br[stream].peekByteFast()>>shift].entry
v = single[uint8(br[stream].value>>shift)].entry
v2 = single[uint8(br[stream2].value>>shift)].entry
br[stream].bitsRead += uint8(v)
br[stream].value <<= v & 63
br[stream2].bitsRead += uint8(v2)
br[stream2].value <<= v2 & 63
buf[off+bufoff*stream+2] = uint8(v >> 8)
br[stream].advance(uint8(v))
v2 = single[br[stream2].peekByteFast()>>shift].entry
buf[off+bufoff*stream2+2] = uint8(v2 >> 8)
br[stream2].advance(uint8(v2))
v = single[br[stream].peekByteFast()>>shift].entry
v = single[uint8(br[stream].value>>shift)].entry
v2 = single[uint8(br[stream2].value>>shift)].entry
br[stream].bitsRead += uint8(v)
br[stream].value <<= v & 63
br[stream2].bitsRead += uint8(v2)
br[stream2].value <<= v2 & 63
buf[off+bufoff*stream+3] = uint8(v >> 8)
br[stream].advance(uint8(v))
v2 = single[br[stream2].peekByteFast()>>shift].entry
buf[off+bufoff*stream2+3] = uint8(v2 >> 8)
br[stream2].advance(uint8(v2))
}
off += 4
@@ -1280,7 +1296,7 @@ func (d *Decoder) decompress4X8bitExactly(dst, src []byte) ([]byte, error) {
}
// Read value and increment offset.
v := single[br.peekByteFast()>>shift].entry
v := single[br.peekByteFast()].entry
nBits := uint8(v)
br.advance(nBits)
bitsLeft -= int(nBits)

256
vendor/github.com/klauspost/compress/s2/README.md сгенерированный поставляемый
Просмотреть файл

@@ -20,6 +20,7 @@ This is important, so you don't have to worry about spending CPU cycles on alrea
* Concurrent stream compression
* Faster decompression, even for Snappy compatible content
* Ability to quickly skip forward in compressed stream
* Random seeking with indexes
* Compatible with reading Snappy compressed content
* Smaller block size overhead on incompressible blocks
* Block concatenation
@@ -29,8 +30,8 @@ This is important, so you don't have to worry about spending CPU cycles on alrea
## Drawbacks over Snappy
* Not optimized for 32 bit systems.
* Streams use slightly more memory due to larger blocks and concurrency (configurable).
* Not optimized for 32 bit systems
* Streams use slightly more memory due to larger blocks and concurrency (configurable)
# Usage
@@ -141,7 +142,7 @@ Binaries can be downloaded on the [Releases Page](https://github.com/klauspost/c
Installing then requires Go to be installed. To install them, use:
`go install github.com/klauspost/compress/s2/cmd/s2c && go install github.com/klauspost/compress/s2/cmd/s2d`
`go install github.com/klauspost/compress/s2/cmd/s2c@latest && go install github.com/klauspost/compress/s2/cmd/s2d@latest`
To build binaries to the current folder use:
@@ -176,6 +177,8 @@ Options:
Compress faster, but with a minor compression loss
-help
Display help
-index
Add seek index (default true)
-o string
Write output to another file. Single input file only
-pad string
@@ -217,11 +220,15 @@ Options:
Display help
-o string
Write output to another file. Single input file only
-q Don't write any output to terminal, except errors
-offset string
Start at offset. Examples: 92, 64K, 256K, 1M, 4M. Requires Index
-q Don't write any output to terminal, except errors
-rm
Delete source file(s) after successful decompression
Delete source file(s) after successful decompression
-safe
Do not overwrite output files
Do not overwrite output files
-tail string
Return last of compressed file. Examples: 92, 64K, 256K, 1M, 4M. Requires Index
-verify
Verify files, but do not write output
```
@@ -633,12 +640,12 @@ Compression and speed is typically a bit better `MaxEncodedLen` is also smaller
Comparison of [`webdevdata.org-2015-01-07-subset`](https://files.klauspost.com/compress/webdevdata.org-2015-01-07-4GB-subset.7z),
53927 files, total input size: 4,014,735,833 bytes. amd64, single goroutine used:
| Encoder | Size | MB/s | Reduction |
|-----------------------|------------|--------|------------
| snappy.Encode | 1128706759 | 725.59 | 71.89% |
| s2.EncodeSnappy | 1093823291 | 899.16 | 72.75% |
| s2.EncodeSnappyBetter | 1001158548 | 578.49 | 75.06% |
| s2.EncodeSnappyBest | 944507998 | 66.00 | 76.47% |
| Encoder | Size | MB/s | Reduction |
|-----------------------|------------|------------|------------
| snappy.Encode | 1128706759 | 725.59 | 71.89% |
| s2.EncodeSnappy | 1093823291 | **899.16** | 72.75% |
| s2.EncodeSnappyBetter | 1001158548 | 578.49 | 75.06% |
| s2.EncodeSnappyBest | 944507998 | 66.00 | **76.47%**|
## Streams
@@ -649,11 +656,11 @@ Comparison of different streams, AMD Ryzen 3950x, 16 cores. Size and throughput:
| File | snappy.NewWriter | S2 Snappy | S2 Snappy, Better | S2 Snappy, Best |
|-----------------------------|--------------------------|---------------------------|--------------------------|-------------------------|
| nyc-taxi-data-10M.csv | 1316042016 - 517.54MB/s | 1307003093 - 8406.29MB/s | 1174534014 - 4984.35MB/s | 1115904679 - 177.81MB/s |
| enwik10 | 5088294643 - 433.45MB/s | 5175840939 - 8454.52MB/s | 4560784526 - 4403.10MB/s | 4340299103 - 159.71MB/s |
| 10gb.tar | 6056946612 - 703.25MB/s | 6208571995 - 9035.75MB/s | 5741646126 - 2402.08MB/s | 5548973895 - 171.17MB/s |
| github-june-2days-2019.json | 1525176492 - 908.11MB/s | 1476519054 - 12625.93MB/s | 1400547532 - 6163.61MB/s | 1321887137 - 200.71MB/s |
| consensus.db.10gb | 5412897703 - 1054.38MB/s | 5354073487 - 12634.82MB/s | 5335069899 - 2472.23MB/s | 5201000954 - 166.32MB/s |
| nyc-taxi-data-10M.csv | 1316042016 - 539.47MB/s | 1307003093 - 10132.73MB/s | 1174534014 - 5002.44MB/s | 1115904679 - 177.97MB/s |
| enwik10 (xml) | 5088294643 - 451.13MB/s | 5175840939 - 9440.69MB/s | 4560784526 - 4487.21MB/s | 4340299103 - 158.92MB/s |
| 10gb.tar (mixed) | 6056946612 - 729.73MB/s | 6208571995 - 9978.05MB/s | 5741646126 - 4919.98MB/s | 5548973895 - 180.44MB/s |
| github-june-2days-2019.json | 1525176492 - 933.00MB/s | 1476519054 - 13150.12MB/s | 1400547532 - 5803.40MB/s | 1321887137 - 204.29MB/s |
| consensus.db.10gb (db) | 5412897703 - 1102.14MB/s | 5354073487 - 13562.91MB/s | 5335069899 - 5294.73MB/s | 5201000954 - 175.72MB/s |
# Decompression
@@ -679,7 +686,220 @@ The 10 byte 'stream identifier' of the second stream can optionally be stripped,
Blocks can be concatenated using the `ConcatBlocks` function.
Snappy blocks/streams can safely be concatenated with S2 blocks and streams.
Snappy blocks/streams can safely be concatenated with S2 blocks and streams.
Streams with indexes (see below) will currently not work on concatenated streams.
# Stream Seek Index
S2 and Snappy streams can have indexes. These indexes will allow random seeking within the compressed data.
The index can either be appended to the stream as a skippable block or returned for separate storage.
When the index is appended to a stream it will be skipped by regular decoders,
so the output remains compatible with other decoders.
## Creating an Index
To automatically add an index to a stream, add `WriterAddIndex()` option to your writer.
Then the index will be added to the stream when `Close()` is called.
```
// Add Index to stream...
enc := s2.NewWriter(w, s2.WriterAddIndex())
io.Copy(enc, r)
enc.Close()
```
If you want to store the index separately, you can use `CloseIndex()` instead of the regular `Close()`.
This will return the index. Note that `CloseIndex()` should only be called once, and you shouldn't call `Close()`.
```
// Get index for separate storage...
enc := s2.NewWriter(w)
io.Copy(enc, r)
index, err := enc.CloseIndex()
```
The `index` can then be used needing to read from the stream.
This means the index can be used without needing to seek to the end of the stream
or for manually forwarding streams. See below.
Finally, an existing S2/Snappy stream can be indexed using the `s2.IndexStream(r io.Reader)` function.
## Using Indexes
To use indexes there is a `ReadSeeker(random bool, index []byte) (*ReadSeeker, error)` function available.
Calling ReadSeeker will return an [io.ReadSeeker](https://pkg.go.dev/io#ReadSeeker) compatible version of the reader.
If 'random' is specified the returned io.Seeker can be used for random seeking, otherwise only forward seeking is supported.
Enabling random seeking requires the original input to support the [io.Seeker](https://pkg.go.dev/io#Seeker) interface.
```
dec := s2.NewReader(r)
rs, err := dec.ReadSeeker(false, nil)
rs.Seek(wantOffset, io.SeekStart)
```
Get a seeker to seek forward. Since no index is provided, the index is read from the stream.
This requires that an index was added and that `r` supports the [io.Seeker](https://pkg.go.dev/io#Seeker) interface.
A custom index can be specified which will be used if supplied.
When using a custom index, it will not be read from the input stream.
```
dec := s2.NewReader(r)
rs, err := dec.ReadSeeker(false, index)
rs.Seek(wantOffset, io.SeekStart)
```
This will read the index from `index`. Since we specify non-random (forward only) seeking `r` does not have to be an io.Seeker
```
dec := s2.NewReader(r)
rs, err := dec.ReadSeeker(true, index)
rs.Seek(wantOffset, io.SeekStart)
```
Finally, since we specify that we want to do random seeking `r` must be an io.Seeker.
The returned [ReadSeeker](https://pkg.go.dev/github.com/klauspost/compress/s2#ReadSeeker) contains a shallow reference to the existing Reader,
meaning changes performed to one is reflected in the other.
To check if a stream contains an index at the end, the `(*Index).LoadStream(rs io.ReadSeeker) error` can be used.
## Manually Forwarding Streams
Indexes can also be read outside the decoder using the [Index](https://pkg.go.dev/github.com/klauspost/compress/s2#Index) type.
This can be used for parsing indexes, either separate or in streams.
In some cases it may not be possible to serve a seekable stream.
This can for instance be an HTTP stream, where the Range request
is sent at the start of the stream.
With a little bit of extra code it is still possible to use indexes
to forward to specific offset with a single forward skip.
It is possible to load the index manually like this:
```
var index s2.Index
_, err = index.Load(idxBytes)
```
This can be used to figure out how much to offset the compressed stream:
```
compressedOffset, uncompressedOffset, err := index.Find(wantOffset)
```
The `compressedOffset` is the number of bytes that should be skipped
from the beginning of the compressed file.
The `uncompressedOffset` will then be offset of the uncompressed bytes returned
when decoding from that position. This will always be <= wantOffset.
When creating a decoder it must be specified that it should *not* expect a stream identifier
at the beginning of the stream. Assuming the io.Reader `r` has been forwarded to `compressedOffset`
we create the decoder like this:
```
dec := s2.NewReader(r, s2.ReaderIgnoreStreamIdentifier())
```
We are not completely done. We still need to forward the stream the uncompressed bytes we didn't want.
This is done using the regular "Skip" function:
```
err = dec.Skip(wantOffset - uncompressedOffset)
```
This will ensure that we are at exactly the offset we want, and reading from `dec` will start at the requested offset.
## Index Format:
Each block is structured as a snappy skippable block, with the chunk ID 0x99.
The block can be read from the front, but contains information so it can be read from the back as well.
Numbers are stored as fixed size little endian values or [zigzag encoded](https://developers.google.com/protocol-buffers/docs/encoding#signed_integers) [base 128 varints](https://developers.google.com/protocol-buffers/docs/encoding),
with un-encoded value length of 64 bits, unless other limits are specified.
| Content | Format |
|---------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------|
| ID, `[1]byte` | Always 0x99. |
| Data Length, `[3]byte` | 3 byte little-endian length of the chunk in bytes, following this. |
| Header `[6]byte` | Header, must be `[115, 50, 105, 100, 120, 0]` or in text: "s2idx\x00". |
| UncompressedSize, Varint | Total Uncompressed size. |
| CompressedSize, Varint | Total Compressed size if known. Should be -1 if unknown. |
| EstBlockSize, Varint | Block Size, used for guessing uncompressed offsets. Must be >= 0. |
| Entries, Varint | Number of Entries in index, must be < 65536 and >=0. |
| HasUncompressedOffsets `byte` | 0 if no uncompressed offsets are present, 1 if present. Other values are invalid. |
| UncompressedOffsets, [Entries]VarInt | Uncompressed offsets. See below how to decode. |
| CompressedOffsets, [Entries]VarInt | Compressed offsets. See below how to decode. |
| Block Size, `[4]byte` | Little Endian total encoded size (including header and trailer). Can be used for searching backwards to start of block. |
| Trailer `[6]byte` | Trailer, must be `[0, 120, 100, 105, 50, 115]` or in text: "\x00xdi2s". Can be used for identifying block from end of stream. |
For regular streams the uncompressed offsets are fully predictable,
so `HasUncompressedOffsets` allows to specify that compressed blocks all have
exactly `EstBlockSize` bytes of uncompressed content.
Entries *must* be in order, starting with the lowest offset,
and there *must* be no uncompressed offset duplicates.
Entries *may* point to the start of a skippable block,
but it is then not allowed to also have an entry for the next block since
that would give an uncompressed offset duplicate.
There is no requirement for all blocks to be represented in the index.
In fact there is a maximum of 65536 block entries in an index.
The writer can use any method to reduce the number of entries.
An implicit block start at 0,0 can be assumed.
### Decoding entries:
```
// Read Uncompressed entries.
// Each assumes EstBlockSize delta from previous.
for each entry {
uOff = 0
if HasUncompressedOffsets == 1 {
uOff = ReadVarInt // Read value from stream
}
// Except for the first entry, use previous values.
if entryNum == 0 {
entry[entryNum].UncompressedOffset = uOff
continue
}
// Uncompressed uses previous offset and adds EstBlockSize
entry[entryNum].UncompressedOffset = entry[entryNum-1].UncompressedOffset + EstBlockSize
}
// Guess that the first block will be 50% of uncompressed size.
// Integer truncating division must be used.
CompressGuess := EstBlockSize / 2
// Read Compressed entries.
// Each assumes CompressGuess delta from previous.
// CompressGuess is adjusted for each value.
for each entry {
cOff = ReadVarInt // Read value from stream
// Except for the first entry, use previous values.
if entryNum == 0 {
entry[entryNum].CompressedOffset = cOff
continue
}
// Compressed uses previous and our estimate.
entry[entryNum].CompressedOffset = entry[entryNum-1].CompressedOffset + CompressGuess + cOff
// Adjust compressed offset for next loop, integer truncating division must be used.
CompressGuess += cOff/2
}
```
# Format Extensions

229
vendor/github.com/klauspost/compress/s2/decode.go сгенерированный поставляемый
Просмотреть файл

@@ -8,7 +8,9 @@ package s2
import (
"encoding/binary"
"errors"
"fmt"
"io"
"io/ioutil"
)
var (
@@ -22,6 +24,16 @@ var (
ErrUnsupported = errors.New("s2: unsupported input")
)
// ErrCantSeek is returned if the stream cannot be seeked.
type ErrCantSeek struct {
Reason string
}
// Error returns the error as string.
func (e ErrCantSeek) Error() string {
return fmt.Sprintf("s2: Can't seek because %s", e.Reason)
}
// DecodedLen returns the length of the decoded block.
func DecodedLen(src []byte) (int, error) {
v, _, err := decodedLen(src)
@@ -88,6 +100,7 @@ func NewReader(r io.Reader, opts ...ReaderOption) *Reader {
} else {
nr.buf = make([]byte, MaxEncodedLen(defaultBlockSize)+checksumSize)
}
nr.readHeader = nr.ignoreStreamID
nr.paramsOK = true
return &nr
}
@@ -131,12 +144,41 @@ func ReaderAllocBlock(blockSize int) ReaderOption {
}
}
// ReaderIgnoreStreamIdentifier will make the reader skip the expected
// stream identifier at the beginning of the stream.
// This can be used when serving a stream that has been forwarded to a specific point.
func ReaderIgnoreStreamIdentifier() ReaderOption {
return func(r *Reader) error {
r.ignoreStreamID = true
return nil
}
}
// ReaderSkippableCB will register a callback for chuncks with the specified ID.
// ID must be a Reserved skippable chunks ID, 0x80-0xfd (inclusive).
// For each chunk with the ID, the callback is called with the content.
// Any returned non-nil error will abort decompression.
// Only one callback per ID is supported, latest sent will be used.
func ReaderSkippableCB(id uint8, fn func(r io.Reader) error) ReaderOption {
return func(r *Reader) error {
if id < 0x80 || id > 0xfd {
return fmt.Errorf("ReaderSkippableCB: Invalid id provided, must be 0x80-0xfd (inclusive)")
}
r.skippableCB[id] = fn
return nil
}
}
// Reader is an io.Reader that can read Snappy-compressed bytes.
type Reader struct {
r io.Reader
err error
decoded []byte
buf []byte
r io.Reader
err error
decoded []byte
buf []byte
skippableCB [0x80]func(r io.Reader) error
blockStart int64 // Uncompressed offset at start of current.
index *Index
// decoded[i:j] contains decoded bytes that have not yet been passed on.
i, j int
// maximum block size allowed.
@@ -144,10 +186,11 @@ type Reader struct {
// maximum expected buffer size.
maxBufSize int
// alloc a buffer this size if > 0.
lazyBuf int
readHeader bool
paramsOK bool
snappyFrame bool
lazyBuf int
readHeader bool
paramsOK bool
snappyFrame bool
ignoreStreamID bool
}
// ensureBufferSize will ensure that the buffer can take at least n bytes.
@@ -172,11 +215,12 @@ func (r *Reader) Reset(reader io.Reader) {
if !r.paramsOK {
return
}
r.index = nil
r.r = reader
r.err = nil
r.i = 0
r.j = 0
r.readHeader = false
r.readHeader = r.ignoreStreamID
}
func (r *Reader) readFull(p []byte, allowEOF bool) (ok bool) {
@@ -189,11 +233,24 @@ func (r *Reader) readFull(p []byte, allowEOF bool) (ok bool) {
return true
}
// skipN will skip n bytes.
// skippable will skip n bytes.
// If the supplied reader supports seeking that is used.
// tmp is used as a temporary buffer for reading.
// The supplied slice does not need to be the size of the read.
func (r *Reader) skipN(tmp []byte, n int, allowEOF bool) (ok bool) {
func (r *Reader) skippable(tmp []byte, n int, allowEOF bool, id uint8) (ok bool) {
if id < 0x80 {
r.err = fmt.Errorf("interbal error: skippable id < 0x80")
return false
}
if fn := r.skippableCB[id-0x80]; fn != nil {
rd := io.LimitReader(r.r, int64(n))
r.err = fn(rd)
if r.err != nil {
return false
}
_, r.err = io.CopyBuffer(ioutil.Discard, rd, tmp)
return r.err == nil
}
if rs, ok := r.r.(io.ReadSeeker); ok {
_, err := rs.Seek(int64(n), io.SeekCurrent)
if err == nil {
@@ -247,6 +304,7 @@ func (r *Reader) Read(p []byte) (int, error) {
// https://github.com/google/snappy/blob/master/framing_format.txt
switch chunkType {
case chunkTypeCompressedData:
r.blockStart += int64(r.j)
// Section 4.2. Compressed data (chunk type 0x00).
if chunkLen < checksumSize {
r.err = ErrCorrupt
@@ -294,6 +352,7 @@ func (r *Reader) Read(p []byte) (int, error) {
continue
case chunkTypeUncompressedData:
r.blockStart += int64(r.j)
// Section 4.3. Uncompressed data (chunk type 0x01).
if chunkLen < checksumSize {
r.err = ErrCorrupt
@@ -357,17 +416,20 @@ func (r *Reader) Read(p []byte) (int, error) {
if chunkType <= 0x7f {
// Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f).
// fmt.Printf("ERR chunktype: 0x%x\n", chunkType)
r.err = ErrUnsupported
return 0, r.err
}
// Section 4.4 Padding (chunk type 0xfe).
// Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd).
if chunkLen > maxBlockSize {
if chunkLen > maxChunkSize {
// fmt.Printf("ERR chunkLen: 0x%x\n", chunkLen)
r.err = ErrUnsupported
return 0, r.err
}
if !r.skipN(r.buf, chunkLen, false) {
// fmt.Printf("skippable: ID: 0x%x, len: 0x%x\n", chunkType, chunkLen)
if !r.skippable(r.buf, chunkLen, false, chunkType) {
return 0, r.err
}
}
@@ -396,7 +458,7 @@ func (r *Reader) Skip(n int64) error {
return nil
}
n -= int64(r.j - r.i)
r.i, r.j = 0, 0
r.i = r.j
}
// Buffer empty; read blocks until we have content.
@@ -420,6 +482,7 @@ func (r *Reader) Skip(n int64) error {
// https://github.com/google/snappy/blob/master/framing_format.txt
switch chunkType {
case chunkTypeCompressedData:
r.blockStart += int64(r.j)
// Section 4.2. Compressed data (chunk type 0x00).
if chunkLen < checksumSize {
r.err = ErrCorrupt
@@ -468,6 +531,7 @@ func (r *Reader) Skip(n int64) error {
r.i, r.j = 0, dLen
continue
case chunkTypeUncompressedData:
r.blockStart += int64(r.j)
// Section 4.3. Uncompressed data (chunk type 0x01).
if chunkLen < checksumSize {
r.err = ErrCorrupt
@@ -528,19 +592,138 @@ func (r *Reader) Skip(n int64) error {
r.err = ErrUnsupported
return r.err
}
if chunkLen > maxBlockSize {
if chunkLen > maxChunkSize {
r.err = ErrUnsupported
return r.err
}
// Section 4.4 Padding (chunk type 0xfe).
// Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd).
if !r.skipN(r.buf, chunkLen, false) {
if !r.skippable(r.buf, chunkLen, false, chunkType) {
return r.err
}
}
return nil
}
// ReadSeeker provides random or forward seeking in compressed content.
// See Reader.ReadSeeker
type ReadSeeker struct {
*Reader
}
// ReadSeeker will return an io.ReadSeeker compatible version of the reader.
// If 'random' is specified the returned io.Seeker can be used for
// random seeking, otherwise only forward seeking is supported.
// Enabling random seeking requires the original input to support
// the io.Seeker interface.
// A custom index can be specified which will be used if supplied.
// When using a custom index, it will not be read from the input stream.
// The returned ReadSeeker contains a shallow reference to the existing Reader,
// meaning changes performed to one is reflected in the other.
func (r *Reader) ReadSeeker(random bool, index []byte) (*ReadSeeker, error) {
// Read index if provided.
if len(index) != 0 {
if r.index == nil {
r.index = &Index{}
}
if _, err := r.index.Load(index); err != nil {
return nil, ErrCantSeek{Reason: "loading index returned: " + err.Error()}
}
}
// Check if input is seekable
rs, ok := r.r.(io.ReadSeeker)
if !ok {
if !random {
return &ReadSeeker{Reader: r}, nil
}
return nil, ErrCantSeek{Reason: "input stream isn't seekable"}
}
if r.index != nil {
// Seekable and index, ok...
return &ReadSeeker{Reader: r}, nil
}
// Load from stream.
r.index = &Index{}
// Read current position.
pos, err := rs.Seek(0, io.SeekCurrent)
if err != nil {
return nil, ErrCantSeek{Reason: "seeking input returned: " + err.Error()}
}
err = r.index.LoadStream(rs)
if err != nil {
if err == ErrUnsupported {
return nil, ErrCantSeek{Reason: "input stream does not contain an index"}
}
return nil, ErrCantSeek{Reason: "reading index returned: " + err.Error()}
}
// reset position.
_, err = rs.Seek(pos, io.SeekStart)
if err != nil {
return nil, ErrCantSeek{Reason: "seeking input returned: " + err.Error()}
}
return &ReadSeeker{Reader: r}, nil
}
// Seek allows seeking in compressed data.
func (r *ReadSeeker) Seek(offset int64, whence int) (int64, error) {
if r.err != nil {
return 0, r.err
}
if offset == 0 && whence == io.SeekCurrent {
return r.blockStart + int64(r.i), nil
}
if !r.readHeader {
// Make sure we read the header.
_, r.err = r.Read([]byte{})
}
rs, ok := r.r.(io.ReadSeeker)
if r.index == nil || !ok {
if whence == io.SeekCurrent && offset >= 0 {
err := r.Skip(offset)
return r.blockStart + int64(r.i), err
}
if whence == io.SeekStart && offset >= r.blockStart+int64(r.i) {
err := r.Skip(offset - r.blockStart - int64(r.i))
return r.blockStart + int64(r.i), err
}
return 0, ErrUnsupported
}
switch whence {
case io.SeekCurrent:
offset += r.blockStart + int64(r.i)
case io.SeekEnd:
offset = -offset
}
c, u, err := r.index.Find(offset)
if err != nil {
return r.blockStart + int64(r.i), err
}
// Seek to next block
_, err = rs.Seek(c, io.SeekStart)
if err != nil {
return 0, err
}
if offset < 0 {
offset = r.index.TotalUncompressed + offset
}
r.i = r.j // Remove rest of current block.
if u < offset {
// Forward inside block
return offset, r.Skip(offset - u)
}
return offset, nil
}
// ReadByte satisfies the io.ByteReader interface.
func (r *Reader) ReadByte() (byte, error) {
if r.err != nil {
@@ -563,3 +746,17 @@ func (r *Reader) ReadByte() (byte, error) {
}
return 0, io.ErrNoProgress
}
// SkippableCB will register a callback for chunks with the specified ID.
// ID must be a Reserved skippable chunks ID, 0x80-0xfe (inclusive).
// For each chunk with the ID, the callback is called with the content.
// Any returned non-nil error will abort decompression.
// Only one callback per ID is supported, latest sent will be used.
// Sending a nil function will disable previous callbacks.
func (r *Reader) SkippableCB(id uint8, fn func(r io.Reader) error) error {
if id < 0x80 || id > chunkTypePadding {
return fmt.Errorf("ReaderSkippableCB: Invalid id provided, must be 0x80-0xfe (inclusive)")
}
r.skippableCB[id] = fn
return nil
}

249
vendor/github.com/klauspost/compress/s2/encode.go сгенерированный поставляемый
Просмотреть файл

@@ -395,23 +395,26 @@ type Writer struct {
// ibuf is a buffer for the incoming (uncompressed) bytes.
ibuf []byte
blockSize int
obufLen int
concurrency int
written int64
output chan chan result
buffers sync.Pool
pad int
blockSize int
obufLen int
concurrency int
written int64
uncompWritten int64 // Bytes sent to compression
output chan chan result
buffers sync.Pool
pad int
writer io.Writer
randSrc io.Reader
writerWg sync.WaitGroup
index Index
// wroteStreamHeader is whether we have written the stream header.
wroteStreamHeader bool
paramsOK bool
snappy bool
flushOnWrite bool
appendIndex bool
level uint8
}
@@ -422,7 +425,11 @@ const (
levelBest
)
type result []byte
type result struct {
b []byte
// Uncompressed start offset
startOffset int64
}
// err returns the previously set error.
// If no error has been set it is set to err if not nil.
@@ -454,6 +461,9 @@ func (w *Writer) Reset(writer io.Writer) {
w.wroteStreamHeader = false
w.written = 0
w.writer = writer
w.uncompWritten = 0
w.index.reset(w.blockSize)
// If we didn't get a writer, stop here.
if writer == nil {
return
@@ -474,7 +484,8 @@ func (w *Writer) Reset(writer io.Writer) {
// Get a queued write.
for write := range toWrite {
// Wait for the data to be available.
in := <-write
input := <-write
in := input.b
if len(in) > 0 {
if w.err(nil) == nil {
// Don't expose data from previous buffers.
@@ -485,11 +496,12 @@ func (w *Writer) Reset(writer io.Writer) {
err = io.ErrShortBuffer
}
_ = w.err(err)
w.err(w.index.add(w.written, input.startOffset))
w.written += int64(n)
}
}
if cap(in) >= w.obufLen {
w.buffers.Put([]byte(in))
w.buffers.Put(in)
}
// close the incoming write request.
// This can be used for synchronizing flushes.
@@ -500,6 +512,9 @@ func (w *Writer) Reset(writer io.Writer) {
// Write satisfies the io.Writer interface.
func (w *Writer) Write(p []byte) (nRet int, errRet error) {
if err := w.err(nil); err != nil {
return 0, err
}
if w.flushOnWrite {
return w.write(p)
}
@@ -535,6 +550,9 @@ func (w *Writer) Write(p []byte) (nRet int, errRet error) {
// The return value n is the number of bytes read.
// Any error except io.EOF encountered during the read is also returned.
func (w *Writer) ReadFrom(r io.Reader) (n int64, err error) {
if err := w.err(nil); err != nil {
return 0, err
}
if len(w.ibuf) > 0 {
err := w.Flush()
if err != nil {
@@ -577,6 +595,85 @@ func (w *Writer) ReadFrom(r io.Reader) (n int64, err error) {
return n, w.err(nil)
}
// AddSkippableBlock will add a skippable block to the stream.
// The ID must be 0x80-0xfe (inclusive).
// Length of the skippable block must be <= 16777215 bytes.
func (w *Writer) AddSkippableBlock(id uint8, data []byte) (err error) {
if err := w.err(nil); err != nil {
return err
}
if len(data) == 0 {
return nil
}
if id < 0x80 || id > chunkTypePadding {
return fmt.Errorf("invalid skippable block id %x", id)
}
if len(data) > maxChunkSize {
return fmt.Errorf("skippable block excessed maximum size")
}
var header [4]byte
chunkLen := 4 + len(data)
header[0] = id
header[1] = uint8(chunkLen >> 0)
header[2] = uint8(chunkLen >> 8)
header[3] = uint8(chunkLen >> 16)
if w.concurrency == 1 {
write := func(b []byte) error {
n, err := w.writer.Write(b)
if err = w.err(err); err != nil {
return err
}
if n != len(data) {
return w.err(io.ErrShortWrite)
}
w.written += int64(n)
return w.err(nil)
}
if !w.wroteStreamHeader {
w.wroteStreamHeader = true
if w.snappy {
if err := write([]byte(magicChunkSnappy)); err != nil {
return err
}
} else {
if err := write([]byte(magicChunk)); err != nil {
return err
}
}
}
if err := write(header[:]); err != nil {
return err
}
if err := write(data); err != nil {
return err
}
}
// Create output...
if !w.wroteStreamHeader {
w.wroteStreamHeader = true
hWriter := make(chan result)
w.output <- hWriter
if w.snappy {
hWriter <- result{startOffset: w.uncompWritten, b: []byte(magicChunkSnappy)}
} else {
hWriter <- result{startOffset: w.uncompWritten, b: []byte(magicChunk)}
}
}
// Copy input.
inbuf := w.buffers.Get().([]byte)[:4]
copy(inbuf, header[:])
inbuf = append(inbuf, data...)
output := make(chan result, 1)
// Queue output.
w.output <- output
output <- result{startOffset: w.uncompWritten, b: inbuf}
return nil
}
// EncodeBuffer will add a buffer to the stream.
// This is the fastest way to encode a stream,
// but the input buffer cannot be written to by the caller
@@ -614,9 +711,9 @@ func (w *Writer) EncodeBuffer(buf []byte) (err error) {
hWriter := make(chan result)
w.output <- hWriter
if w.snappy {
hWriter <- []byte(magicChunkSnappy)
hWriter <- result{startOffset: w.uncompWritten, b: []byte(magicChunkSnappy)}
} else {
hWriter <- []byte(magicChunk)
hWriter <- result{startOffset: w.uncompWritten, b: []byte(magicChunk)}
}
}
@@ -632,6 +729,10 @@ func (w *Writer) EncodeBuffer(buf []byte) (err error) {
output := make(chan result)
// Queue output now, so we keep order.
w.output <- output
res := result{
startOffset: w.uncompWritten,
}
w.uncompWritten += int64(len(uncompressed))
go func() {
checksum := crc(uncompressed)
@@ -664,7 +765,8 @@ func (w *Writer) EncodeBuffer(buf []byte) (err error) {
obuf[7] = uint8(checksum >> 24)
// Queue final output.
output <- obuf
res.b = obuf
output <- res
}()
}
return nil
@@ -708,9 +810,9 @@ func (w *Writer) write(p []byte) (nRet int, errRet error) {
hWriter := make(chan result)
w.output <- hWriter
if w.snappy {
hWriter <- []byte(magicChunkSnappy)
hWriter <- result{startOffset: w.uncompWritten, b: []byte(magicChunkSnappy)}
} else {
hWriter <- []byte(magicChunk)
hWriter <- result{startOffset: w.uncompWritten, b: []byte(magicChunk)}
}
}
@@ -731,6 +833,11 @@ func (w *Writer) write(p []byte) (nRet int, errRet error) {
output := make(chan result)
// Queue output now, so we keep order.
w.output <- output
res := result{
startOffset: w.uncompWritten,
}
w.uncompWritten += int64(len(uncompressed))
go func() {
checksum := crc(uncompressed)
@@ -763,7 +870,8 @@ func (w *Writer) write(p []byte) (nRet int, errRet error) {
obuf[7] = uint8(checksum >> 24)
// Queue final output.
output <- obuf
res.b = obuf
output <- res
// Put unused buffer back in pool.
w.buffers.Put(inbuf)
@@ -793,9 +901,9 @@ func (w *Writer) writeFull(inbuf []byte) (errRet error) {
hWriter := make(chan result)
w.output <- hWriter
if w.snappy {
hWriter <- []byte(magicChunkSnappy)
hWriter <- result{startOffset: w.uncompWritten, b: []byte(magicChunkSnappy)}
} else {
hWriter <- []byte(magicChunk)
hWriter <- result{startOffset: w.uncompWritten, b: []byte(magicChunk)}
}
}
@@ -806,6 +914,11 @@ func (w *Writer) writeFull(inbuf []byte) (errRet error) {
output := make(chan result)
// Queue output now, so we keep order.
w.output <- output
res := result{
startOffset: w.uncompWritten,
}
w.uncompWritten += int64(len(uncompressed))
go func() {
checksum := crc(uncompressed)
@@ -838,7 +951,8 @@ func (w *Writer) writeFull(inbuf []byte) (errRet error) {
obuf[7] = uint8(checksum >> 24)
// Queue final output.
output <- obuf
res.b = obuf
output <- res
// Put unused buffer back in pool.
w.buffers.Put(inbuf)
@@ -912,7 +1026,10 @@ func (w *Writer) writeSync(p []byte) (nRet int, errRet error) {
if n != len(obuf) {
return 0, w.err(io.ErrShortWrite)
}
w.err(w.index.add(w.written, w.uncompWritten))
w.written += int64(n)
w.uncompWritten += int64(len(uncompressed))
if chunkType == chunkTypeUncompressedData {
// Write uncompressed data.
n, err := w.writer.Write(uncompressed)
@@ -961,38 +1078,87 @@ func (w *Writer) Flush() error {
res := make(chan result)
w.output <- res
// Block until this has been picked up.
res <- nil
res <- result{b: nil, startOffset: w.uncompWritten}
// When it is closed, we have flushed.
<-res
return w.err(nil)
}
// Close calls Flush and then closes the Writer.
// Calling Close multiple times is ok.
// Calling Close multiple times is ok,
// but calling CloseIndex after this will make it not return the index.
func (w *Writer) Close() error {
_, err := w.closeIndex(w.appendIndex)
return err
}
// CloseIndex calls Close and returns an index on first call.
// This is not required if you are only adding index to a stream.
func (w *Writer) CloseIndex() ([]byte, error) {
return w.closeIndex(true)
}
func (w *Writer) closeIndex(idx bool) ([]byte, error) {
err := w.Flush()
if w.output != nil {
close(w.output)
w.writerWg.Wait()
w.output = nil
}
if w.err(nil) == nil && w.writer != nil && w.pad > 0 {
add := calcSkippableFrame(w.written, int64(w.pad))
frame, err := skippableFrame(w.ibuf[:0], add, w.randSrc)
if err = w.err(err); err != nil {
return err
}
_, err2 := w.writer.Write(frame)
_ = w.err(err2)
}
_ = w.err(errClosed)
if err == errClosed {
return nil
}
return err
}
const skippableFrameHeader = 4
var index []byte
if w.err(nil) == nil && w.writer != nil {
// Create index.
if idx {
compSize := int64(-1)
if w.pad <= 1 {
compSize = w.written
}
index = w.index.appendTo(w.ibuf[:0], w.uncompWritten, compSize)
// Count as written for padding.
if w.appendIndex {
w.written += int64(len(index))
}
if true {
_, err := w.index.Load(index)
if err != nil {
panic(err)
}
}
}
if w.pad > 1 {
tmp := w.ibuf[:0]
if len(index) > 0 {
// Allocate another buffer.
tmp = w.buffers.Get().([]byte)[:0]
defer w.buffers.Put(tmp)
}
add := calcSkippableFrame(w.written, int64(w.pad))
frame, err := skippableFrame(tmp, add, w.randSrc)
if err = w.err(err); err != nil {
return nil, err
}
n, err2 := w.writer.Write(frame)
if err2 == nil && n != len(frame) {
err2 = io.ErrShortWrite
}
_ = w.err(err2)
}
if len(index) > 0 && w.appendIndex {
n, err2 := w.writer.Write(index)
if err2 == nil && n != len(index) {
err2 = io.ErrShortWrite
}
_ = w.err(err2)
}
}
err = w.err(errClosed)
if err == errClosed {
return index, nil
}
return nil, err
}
// calcSkippableFrame will return a total size to be added for written
// to be divisible by multiple.
@@ -1057,6 +1223,15 @@ func WriterConcurrency(n int) WriterOption {
}
}
// WriterAddIndex will append an index to the end of a stream
// when it is closed.
func WriterAddIndex() WriterOption {
return func(w *Writer) error {
w.appendIndex = true
return nil
}
}
// WriterBetterCompression will enable better compression.
// EncodeBetter compresses better than Encode but typically with a
// 10-40% speed decrease on both compression and decompression.

525
vendor/github.com/klauspost/compress/s2/index.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,525 @@
// Copyright (c) 2022+ Klaus Post. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package s2
import (
"bytes"
"encoding/binary"
"encoding/json"
"fmt"
"io"
)
const (
S2IndexHeader = "s2idx\x00"
S2IndexTrailer = "\x00xdi2s"
maxIndexEntries = 1 << 16
)
// Index represents an S2/Snappy index.
type Index struct {
TotalUncompressed int64 // Total Uncompressed size if known. Will be -1 if unknown.
TotalCompressed int64 // Total Compressed size if known. Will be -1 if unknown.
info []struct {
compressedOffset int64
uncompressedOffset int64
}
estBlockUncomp int64
}
func (i *Index) reset(maxBlock int) {
i.estBlockUncomp = int64(maxBlock)
i.TotalCompressed = -1
i.TotalUncompressed = -1
if len(i.info) > 0 {
i.info = i.info[:0]
}
}
// allocInfos will allocate an empty slice of infos.
func (i *Index) allocInfos(n int) {
if n > maxIndexEntries {
panic("n > maxIndexEntries")
}
i.info = make([]struct {
compressedOffset int64
uncompressedOffset int64
}, 0, n)
}
// add an uncompressed and compressed pair.
// Entries must be sent in order.
func (i *Index) add(compressedOffset, uncompressedOffset int64) error {
if i == nil {
return nil
}
lastIdx := len(i.info) - 1
if lastIdx >= 0 {
latest := i.info[lastIdx]
if latest.uncompressedOffset == uncompressedOffset {
// Uncompressed didn't change, don't add entry,
// but update start index.
latest.compressedOffset = compressedOffset
i.info[lastIdx] = latest
return nil
}
if latest.uncompressedOffset > uncompressedOffset {
return fmt.Errorf("internal error: Earlier uncompressed received (%d > %d)", latest.uncompressedOffset, uncompressedOffset)
}
if latest.compressedOffset > compressedOffset {
return fmt.Errorf("internal error: Earlier compressed received (%d > %d)", latest.uncompressedOffset, uncompressedOffset)
}
}
i.info = append(i.info, struct {
compressedOffset int64
uncompressedOffset int64
}{compressedOffset: compressedOffset, uncompressedOffset: uncompressedOffset})
return nil
}
// Find the offset at or before the wanted (uncompressed) offset.
// If offset is 0 or positive it is the offset from the beginning of the file.
// If the uncompressed size is known, the offset must be within the file.
// If an offset outside the file is requested io.ErrUnexpectedEOF is returned.
// If the offset is negative, it is interpreted as the distance from the end of the file,
// where -1 represents the last byte.
// If offset from the end of the file is requested, but size is unknown,
// ErrUnsupported will be returned.
func (i *Index) Find(offset int64) (compressedOff, uncompressedOff int64, err error) {
if i.TotalUncompressed < 0 {
return 0, 0, ErrCorrupt
}
if offset < 0 {
offset = i.TotalUncompressed + offset
if offset < 0 {
return 0, 0, io.ErrUnexpectedEOF
}
}
if offset > i.TotalUncompressed {
return 0, 0, io.ErrUnexpectedEOF
}
for _, info := range i.info {
if info.uncompressedOffset > offset {
break
}
compressedOff = info.compressedOffset
uncompressedOff = info.uncompressedOffset
}
return compressedOff, uncompressedOff, nil
}
// reduce to stay below maxIndexEntries
func (i *Index) reduce() {
if len(i.info) < maxIndexEntries && i.estBlockUncomp >= 1<<20 {
return
}
// Algorithm, keep 1, remove removeN entries...
removeN := (len(i.info) + 1) / maxIndexEntries
src := i.info
j := 0
// Each block should be at least 1MB, but don't reduce below 1000 entries.
for i.estBlockUncomp*(int64(removeN)+1) < 1<<20 && len(i.info)/(removeN+1) > 1000 {
removeN++
}
for idx := 0; idx < len(src); idx++ {
i.info[j] = src[idx]
j++
idx += removeN
}
i.info = i.info[:j]
// Update maxblock estimate.
i.estBlockUncomp += i.estBlockUncomp * int64(removeN)
}
func (i *Index) appendTo(b []byte, uncompTotal, compTotal int64) []byte {
i.reduce()
var tmp [binary.MaxVarintLen64]byte
initSize := len(b)
// We make the start a skippable header+size.
b = append(b, ChunkTypeIndex, 0, 0, 0)
b = append(b, []byte(S2IndexHeader)...)
// Total Uncompressed size
n := binary.PutVarint(tmp[:], uncompTotal)
b = append(b, tmp[:n]...)
// Total Compressed size
n = binary.PutVarint(tmp[:], compTotal)
b = append(b, tmp[:n]...)
// Put EstBlockUncomp size
n = binary.PutVarint(tmp[:], i.estBlockUncomp)
b = append(b, tmp[:n]...)
// Put length
n = binary.PutVarint(tmp[:], int64(len(i.info)))
b = append(b, tmp[:n]...)
// Check if we should add uncompressed offsets
var hasUncompressed byte
for idx, info := range i.info {
if idx == 0 {
if info.uncompressedOffset != 0 {
hasUncompressed = 1
break
}
continue
}
if info.uncompressedOffset != i.info[idx-1].uncompressedOffset+i.estBlockUncomp {
hasUncompressed = 1
break
}
}
b = append(b, hasUncompressed)
// Add each entry
if hasUncompressed == 1 {
for idx, info := range i.info {
uOff := info.uncompressedOffset
if idx > 0 {
prev := i.info[idx-1]
uOff -= prev.uncompressedOffset + (i.estBlockUncomp)
}
n = binary.PutVarint(tmp[:], uOff)
b = append(b, tmp[:n]...)
}
}
// Initial compressed size estimate.
cPredict := i.estBlockUncomp / 2
for idx, info := range i.info {
cOff := info.compressedOffset
if idx > 0 {
prev := i.info[idx-1]
cOff -= prev.compressedOffset + cPredict
// Update compressed size prediction, with half the error.
cPredict += cOff / 2
}
n = binary.PutVarint(tmp[:], cOff)
b = append(b, tmp[:n]...)
}
// Add Total Size.
// Stored as fixed size for easier reading.
binary.LittleEndian.PutUint32(tmp[:], uint32(len(b)-initSize+4+len(S2IndexTrailer)))
b = append(b, tmp[:4]...)
// Trailer
b = append(b, []byte(S2IndexTrailer)...)
// Update size
chunkLen := len(b) - initSize - skippableFrameHeader
b[initSize+1] = uint8(chunkLen >> 0)
b[initSize+2] = uint8(chunkLen >> 8)
b[initSize+3] = uint8(chunkLen >> 16)
//fmt.Printf("chunklen: 0x%x Uncomp:%d, Comp:%d\n", chunkLen, uncompTotal, compTotal)
return b
}
// Load a binary index.
// A zero value Index can be used or a previous one can be reused.
func (i *Index) Load(b []byte) ([]byte, error) {
if len(b) <= 4+len(S2IndexHeader)+len(S2IndexTrailer) {
return b, io.ErrUnexpectedEOF
}
if b[0] != ChunkTypeIndex {
return b, ErrCorrupt
}
chunkLen := int(b[1]) | int(b[2])<<8 | int(b[3])<<16
b = b[4:]
// Validate we have enough...
if len(b) < chunkLen {
return b, io.ErrUnexpectedEOF
}
if !bytes.Equal(b[:len(S2IndexHeader)], []byte(S2IndexHeader)) {
return b, ErrUnsupported
}
b = b[len(S2IndexHeader):]
// Total Uncompressed
if v, n := binary.Varint(b); n <= 0 || v < 0 {
return b, ErrCorrupt
} else {
i.TotalUncompressed = v
b = b[n:]
}
// Total Compressed
if v, n := binary.Varint(b); n <= 0 {
return b, ErrCorrupt
} else {
i.TotalCompressed = v
b = b[n:]
}
// Read EstBlockUncomp
if v, n := binary.Varint(b); n <= 0 {
return b, ErrCorrupt
} else {
if v < 0 {
return b, ErrCorrupt
}
i.estBlockUncomp = v
b = b[n:]
}
var entries int
if v, n := binary.Varint(b); n <= 0 {
return b, ErrCorrupt
} else {
if v < 0 || v > maxIndexEntries {
return b, ErrCorrupt
}
entries = int(v)
b = b[n:]
}
if cap(i.info) < entries {
i.allocInfos(entries)
}
i.info = i.info[:entries]
if len(b) < 1 {
return b, io.ErrUnexpectedEOF
}
hasUncompressed := b[0]
b = b[1:]
if hasUncompressed&1 != hasUncompressed {
return b, ErrCorrupt
}
// Add each uncompressed entry
for idx := range i.info {
var uOff int64
if hasUncompressed != 0 {
// Load delta
if v, n := binary.Varint(b); n <= 0 {
return b, ErrCorrupt
} else {
uOff = v
b = b[n:]
}
}
if idx > 0 {
prev := i.info[idx-1].uncompressedOffset
uOff += prev + (i.estBlockUncomp)
if uOff <= prev {
return b, ErrCorrupt
}
}
if uOff < 0 {
return b, ErrCorrupt
}
i.info[idx].uncompressedOffset = uOff
}
// Initial compressed size estimate.
cPredict := i.estBlockUncomp / 2
// Add each compressed entry
for idx := range i.info {
var cOff int64
if v, n := binary.Varint(b); n <= 0 {
return b, ErrCorrupt
} else {
cOff = v
b = b[n:]
}
if idx > 0 {
// Update compressed size prediction, with half the error.
cPredictNew := cPredict + cOff/2
prev := i.info[idx-1].compressedOffset
cOff += prev + cPredict
if cOff <= prev {
return b, ErrCorrupt
}
cPredict = cPredictNew
}
if cOff < 0 {
return b, ErrCorrupt
}
i.info[idx].compressedOffset = cOff
}
if len(b) < 4+len(S2IndexTrailer) {
return b, io.ErrUnexpectedEOF
}
// Skip size...
b = b[4:]
// Check trailer...
if !bytes.Equal(b[:len(S2IndexTrailer)], []byte(S2IndexTrailer)) {
return b, ErrCorrupt
}
return b[len(S2IndexTrailer):], nil
}
// LoadStream will load an index from the end of the supplied stream.
// ErrUnsupported will be returned if the signature cannot be found.
// ErrCorrupt will be returned if unexpected values are found.
// io.ErrUnexpectedEOF is returned if there are too few bytes.
// IO errors are returned as-is.
func (i *Index) LoadStream(rs io.ReadSeeker) error {
// Go to end.
_, err := rs.Seek(-10, io.SeekEnd)
if err != nil {
return err
}
var tmp [10]byte
_, err = io.ReadFull(rs, tmp[:])
if err != nil {
return err
}
// Check trailer...
if !bytes.Equal(tmp[4:4+len(S2IndexTrailer)], []byte(S2IndexTrailer)) {
return ErrUnsupported
}
sz := binary.LittleEndian.Uint32(tmp[:4])
if sz > maxChunkSize+skippableFrameHeader {
return ErrCorrupt
}
_, err = rs.Seek(-int64(sz), io.SeekEnd)
if err != nil {
return err
}
// Read index.
buf := make([]byte, sz)
_, err = io.ReadFull(rs, buf)
if err != nil {
return err
}
_, err = i.Load(buf)
return err
}
// IndexStream will return an index for a stream.
// The stream structure will be checked, but
// data within blocks is not verified.
// The returned index can either be appended to the end of the stream
// or stored separately.
func IndexStream(r io.Reader) ([]byte, error) {
var i Index
var buf [maxChunkSize]byte
var readHeader bool
for {
_, err := io.ReadFull(r, buf[:4])
if err != nil {
if err == io.EOF {
return i.appendTo(nil, i.TotalUncompressed, i.TotalCompressed), nil
}
return nil, err
}
// Start of this chunk.
startChunk := i.TotalCompressed
i.TotalCompressed += 4
chunkType := buf[0]
if !readHeader {
if chunkType != chunkTypeStreamIdentifier {
return nil, ErrCorrupt
}
readHeader = true
}
chunkLen := int(buf[1]) | int(buf[2])<<8 | int(buf[3])<<16
if chunkLen < checksumSize {
return nil, ErrCorrupt
}
i.TotalCompressed += int64(chunkLen)
_, err = io.ReadFull(r, buf[:chunkLen])
if err != nil {
return nil, io.ErrUnexpectedEOF
}
// The chunk types are specified at
// https://github.com/google/snappy/blob/master/framing_format.txt
switch chunkType {
case chunkTypeCompressedData:
// Section 4.2. Compressed data (chunk type 0x00).
// Skip checksum.
dLen, err := DecodedLen(buf[checksumSize:])
if err != nil {
return nil, err
}
if dLen > maxBlockSize {
return nil, ErrCorrupt
}
if i.estBlockUncomp == 0 {
// Use first block for estimate...
i.estBlockUncomp = int64(dLen)
}
err = i.add(startChunk, i.TotalUncompressed)
if err != nil {
return nil, err
}
i.TotalUncompressed += int64(dLen)
continue
case chunkTypeUncompressedData:
n2 := chunkLen - checksumSize
if n2 > maxBlockSize {
return nil, ErrCorrupt
}
if i.estBlockUncomp == 0 {
// Use first block for estimate...
i.estBlockUncomp = int64(n2)
}
err = i.add(startChunk, i.TotalUncompressed)
if err != nil {
return nil, err
}
i.TotalUncompressed += int64(n2)
continue
case chunkTypeStreamIdentifier:
// Section 4.1. Stream identifier (chunk type 0xff).
if chunkLen != len(magicBody) {
return nil, ErrCorrupt
}
if string(buf[:len(magicBody)]) != magicBody {
if string(buf[:len(magicBody)]) != magicBodySnappy {
return nil, ErrCorrupt
}
}
continue
}
if chunkType <= 0x7f {
// Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f).
return nil, ErrUnsupported
}
if chunkLen > maxChunkSize {
return nil, ErrUnsupported
}
// Section 4.4 Padding (chunk type 0xfe).
// Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd).
}
}
// JSON returns the index as JSON text.
func (i *Index) JSON() []byte {
x := struct {
TotalUncompressed int64 `json:"total_uncompressed"` // Total Uncompressed size if known. Will be -1 if unknown.
TotalCompressed int64 `json:"total_compressed"` // Total Compressed size if known. Will be -1 if unknown.
Offsets []struct {
CompressedOffset int64 `json:"compressed"`
UncompressedOffset int64 `json:"uncompressed"`
} `json:"offsets"`
EstBlockUncomp int64 `json:"est_block_uncompressed"`
}{
TotalUncompressed: i.TotalUncompressed,
TotalCompressed: i.TotalCompressed,
EstBlockUncomp: i.estBlockUncomp,
}
for _, v := range i.info {
x.Offsets = append(x.Offsets, struct {
CompressedOffset int64 `json:"compressed"`
UncompressedOffset int64 `json:"uncompressed"`
}{CompressedOffset: v.compressedOffset, UncompressedOffset: v.uncompressedOffset})
}
b, _ := json.MarshalIndent(x, "", " ")
return b
}

4
vendor/github.com/klauspost/compress/s2/s2.go сгенерированный поставляемый
Просмотреть файл

@@ -87,6 +87,9 @@ const (
// minBlockSize is the minimum size of block setting when creating a writer.
minBlockSize = 4 << 10
skippableFrameHeader = 4
maxChunkSize = 1<<24 - 1 // 16777215
// Default block size
defaultBlockSize = 1 << 20
@@ -99,6 +102,7 @@ const (
const (
chunkTypeCompressedData = 0x00
chunkTypeUncompressedData = 0x01
ChunkTypeIndex = 0x99
chunkTypePadding = 0xfe
chunkTypeStreamIdentifier = 0xff
)

15
vendor/github.com/klauspost/compress/zstd/bitreader.go сгенерированный поставляемый
Просмотреть файл

@@ -50,16 +50,23 @@ func (b *bitReader) getBits(n uint8) int {
if n == 0 /*|| b.bitsRead >= 64 */ {
return 0
}
return b.getBitsFast(n)
return int(b.get32BitsFast(n))
}
// getBitsFast requires that at least one bit is requested every time.
// get32BitsFast requires that at least one bit is requested every time.
// There are no checks if the buffer is filled.
func (b *bitReader) getBitsFast(n uint8) int {
func (b *bitReader) get32BitsFast(n uint8) uint32 {
const regMask = 64 - 1
v := uint32((b.value << (b.bitsRead & regMask)) >> ((regMask + 1 - n) & regMask))
b.bitsRead += n
return int(v)
return v
}
func (b *bitReader) get16BitsFast(n uint8) uint16 {
const regMask = 64 - 1
v := uint16((b.value << (b.bitsRead & regMask)) >> ((regMask + 1 - n) & regMask))
b.bitsRead += n
return v
}
// fillFast() will make sure at least 32 bits are available.

22
vendor/github.com/klauspost/compress/zstd/bitwriter.go сгенерированный поставляемый
Просмотреть файл

@@ -38,7 +38,7 @@ func (b *bitWriter) addBits16NC(value uint16, bits uint8) {
b.nBits += bits
}
// addBits32NC will add up to 32 bits.
// addBits32NC will add up to 31 bits.
// It will not check if there is space for them,
// so the caller must ensure that it has flushed recently.
func (b *bitWriter) addBits32NC(value uint32, bits uint8) {
@@ -46,6 +46,26 @@ func (b *bitWriter) addBits32NC(value uint32, bits uint8) {
b.nBits += bits
}
// addBits64NC will add up to 64 bits.
// There must be space for 32 bits.
func (b *bitWriter) addBits64NC(value uint64, bits uint8) {
if bits <= 31 {
b.addBits32Clean(uint32(value), bits)
return
}
b.addBits32Clean(uint32(value), 32)
b.flush32()
b.addBits32Clean(uint32(value>>32), bits-32)
}
// addBits32Clean will add up to 32 bits.
// It will not check if there is space for them.
// The input must not contain more bits than specified.
func (b *bitWriter) addBits32Clean(value uint32, bits uint8) {
b.bitContainer |= uint64(value) << (b.nBits & 63)
b.nBits += bits
}
// addBits16Clean will add up to 16 bits. value may not contain more set bits than indicated.
// It will not check if there is space for them, so the caller must ensure that it has flushed recently.
func (b *bitWriter) addBits16Clean(value uint16, bits uint8) {

24
vendor/github.com/klauspost/compress/zstd/blockdec.go сгенерированный поставляемый
Просмотреть файл

@@ -76,12 +76,11 @@ type blockDec struct {
// Window size of the block.
WindowSize uint64
history chan *history
input chan struct{}
result chan decodeOutput
sequenceBuf []seq
err error
decWG sync.WaitGroup
history chan *history
input chan struct{}
result chan decodeOutput
err error
decWG sync.WaitGroup
// Frame to use for singlethreaded decoding.
// Should not be used by the decoder itself since parent may be another frame.
@@ -512,18 +511,7 @@ func (b *blockDec) decodeCompressed(hist *history) error {
nSeqs = 0x7f00 + int(in[1]) + (int(in[2]) << 8)
in = in[3:]
}
// Allocate sequences
if cap(b.sequenceBuf) < nSeqs {
if b.lowMem {
b.sequenceBuf = make([]seq, nSeqs)
} else {
// Allocate max
b.sequenceBuf = make([]seq, nSeqs, maxSequences)
}
} else {
// Reuse buffer
b.sequenceBuf = b.sequenceBuf[:nSeqs]
}
var seqs = &sequenceDecs{}
if nSeqs > 0 {
if len(in) < 1 {

98
vendor/github.com/klauspost/compress/zstd/blockenc.go сгенерированный поставляемый
Просмотреть файл

@@ -51,7 +51,7 @@ func (b *blockEnc) init() {
if cap(b.literals) < maxCompressedBlockSize {
b.literals = make([]byte, 0, maxCompressedBlockSize)
}
const defSeqs = 200
const defSeqs = 2000
if cap(b.sequences) < defSeqs {
b.sequences = make([]seq, 0, defSeqs)
}
@@ -426,7 +426,7 @@ func fuzzFseEncoder(data []byte) int {
return 0
}
enc := fseEncoder{}
hist := enc.Histogram()[:256]
hist := enc.Histogram()
maxSym := uint8(0)
for i, v := range data {
v = v & 63
@@ -722,52 +722,53 @@ func (b *blockEnc) encode(org []byte, raw, rawAllLits bool) error {
println("Encoded seq", seq, s, "codes:", s.llCode, s.mlCode, s.ofCode, "states:", ll.state, ml.state, of.state, "bits:", llB, mlB, ofB)
}
seq--
if llEnc.maxBits+mlEnc.maxBits+ofEnc.maxBits <= 32 {
// No need to flush (common)
for seq >= 0 {
s = b.sequences[seq]
wr.flush32()
llB, ofB, mlB := llTT[s.llCode], ofTT[s.ofCode], mlTT[s.mlCode]
// tabelog max is 8 for all.
of.encode(ofB)
ml.encode(mlB)
ll.encode(llB)
wr.flush32()
// Store sequences in reverse...
for seq >= 0 {
s = b.sequences[seq]
// We checked that all can stay within 32 bits
wr.addBits32NC(s.litLen, llB.outBits)
wr.addBits32NC(s.matchLen, mlB.outBits)
wr.addBits32NC(s.offset, ofB.outBits)
ofB := ofTT[s.ofCode]
wr.flush32() // tablelog max is below 8 for each, so it will fill max 24 bits.
//of.encode(ofB)
nbBitsOut := (uint32(of.state) + ofB.deltaNbBits) >> 16
dstState := int32(of.state>>(nbBitsOut&15)) + int32(ofB.deltaFindState)
wr.addBits16NC(of.state, uint8(nbBitsOut))
of.state = of.stateTable[dstState]
if debugSequences {
println("Encoded seq", seq, s)
}
// Accumulate extra bits.
outBits := ofB.outBits & 31
extraBits := uint64(s.offset & bitMask32[outBits])
extraBitsN := outBits
seq--
mlB := mlTT[s.mlCode]
//ml.encode(mlB)
nbBitsOut = (uint32(ml.state) + mlB.deltaNbBits) >> 16
dstState = int32(ml.state>>(nbBitsOut&15)) + int32(mlB.deltaFindState)
wr.addBits16NC(ml.state, uint8(nbBitsOut))
ml.state = ml.stateTable[dstState]
outBits = mlB.outBits & 31
extraBits = extraBits<<outBits | uint64(s.matchLen&bitMask32[outBits])
extraBitsN += outBits
llB := llTT[s.llCode]
//ll.encode(llB)
nbBitsOut = (uint32(ll.state) + llB.deltaNbBits) >> 16
dstState = int32(ll.state>>(nbBitsOut&15)) + int32(llB.deltaFindState)
wr.addBits16NC(ll.state, uint8(nbBitsOut))
ll.state = ll.stateTable[dstState]
outBits = llB.outBits & 31
extraBits = extraBits<<outBits | uint64(s.litLen&bitMask32[outBits])
extraBitsN += outBits
wr.flush32()
wr.addBits64NC(extraBits, extraBitsN)
if debugSequences {
println("Encoded seq", seq, s)
}
} else {
for seq >= 0 {
s = b.sequences[seq]
wr.flush32()
llB, ofB, mlB := llTT[s.llCode], ofTT[s.ofCode], mlTT[s.mlCode]
// tabelog max is below 8 for each.
of.encode(ofB)
ml.encode(mlB)
ll.encode(llB)
wr.flush32()
// ml+ll = max 32 bits total
wr.addBits32NC(s.litLen, llB.outBits)
wr.addBits32NC(s.matchLen, mlB.outBits)
wr.flush32()
wr.addBits32NC(s.offset, ofB.outBits)
if debugSequences {
println("Encoded seq", seq, s)
}
seq--
}
seq--
}
ml.flush(mlEnc.actualTableLog)
of.flush(ofEnc.actualTableLog)
@@ -801,14 +802,13 @@ func (b *blockEnc) genCodes() {
// nothing to do
return
}
if len(b.sequences) > math.MaxUint16 {
panic("can only encode up to 64K sequences")
}
// No bounds checks after here:
llH := b.coders.llEnc.Histogram()[:256]
ofH := b.coders.ofEnc.Histogram()[:256]
mlH := b.coders.mlEnc.Histogram()[:256]
llH := b.coders.llEnc.Histogram()
ofH := b.coders.ofEnc.Histogram()
mlH := b.coders.mlEnc.Histogram()
for i := range llH {
llH[i] = 0
}
@@ -820,7 +820,8 @@ func (b *blockEnc) genCodes() {
}
var llMax, ofMax, mlMax uint8
for i, seq := range b.sequences {
for i := range b.sequences {
seq := &b.sequences[i]
v := llCode(seq.litLen)
seq.llCode = v
llH[v]++
@@ -844,7 +845,6 @@ func (b *blockEnc) genCodes() {
panic(fmt.Errorf("mlMax > maxMatchLengthSymbol (%d), matchlen: %d", mlMax, seq.matchLen))
}
}
b.sequences[i] = seq
}
maxCount := func(a []uint32) int {
var max uint32

98
vendor/github.com/klauspost/compress/zstd/decodeheader.go сгенерированный поставляемый
Просмотреть файл

@@ -5,6 +5,7 @@ package zstd
import (
"bytes"
"encoding/binary"
"errors"
"io"
)
@@ -15,18 +16,50 @@ const HeaderMaxSize = 14 + 3
// Header contains information about the first frame and block within that.
type Header struct {
// Window Size the window of data to keep while decoding.
// Will only be set if HasFCS is false.
WindowSize uint64
// SingleSegment specifies whether the data is to be decompressed into a
// single contiguous memory segment.
// It implies that WindowSize is invalid and that FrameContentSize is valid.
SingleSegment bool
// Frame content size.
// Expected size of the entire frame.
FrameContentSize uint64
// WindowSize is the window of data to keep while decoding.
// Will only be set if SingleSegment is false.
WindowSize uint64
// Dictionary ID.
// If 0, no dictionary.
DictionaryID uint32
// HasFCS specifies whether FrameContentSize has a valid value.
HasFCS bool
// FrameContentSize is the expected uncompressed size of the entire frame.
FrameContentSize uint64
// Skippable will be true if the frame is meant to be skipped.
// This implies that FirstBlock.OK is false.
Skippable bool
// SkippableID is the user-specific ID for the skippable frame.
// Valid values are between 0 to 15, inclusive.
SkippableID int
// SkippableSize is the length of the user data to skip following
// the header.
SkippableSize uint32
// HeaderSize is the raw size of the frame header.
//
// For normal frames, it includes the size of the magic number and
// the size of the header (per section 3.1.1.1).
// It does not include the size for any data blocks (section 3.1.1.2) nor
// the size for the trailing content checksum.
//
// For skippable frames, this counts the size of the magic number
// along with the size of the size field of the payload.
// It does not include the size of the skippable payload itself.
// The total frame size is the HeaderSize plus the SkippableSize.
HeaderSize int
// First block information.
FirstBlock struct {
// OK will be set if first block could be decoded.
@@ -51,17 +84,9 @@ type Header struct {
CompressedSize int
}
// Skippable will be true if the frame is meant to be skipped.
// No other information will be populated.
Skippable bool
// If set there is a checksum present for the block content.
// The checksum field at the end is always 4 bytes long.
HasCheckSum bool
// If this is true FrameContentSize will have a valid value
HasFCS bool
SingleSegment bool
}
// Decode the header from the beginning of the stream.
@@ -71,39 +96,46 @@ type Header struct {
// If there isn't enough input, io.ErrUnexpectedEOF is returned.
// The FirstBlock.OK will indicate if enough information was available to decode the first block header.
func (h *Header) Decode(in []byte) error {
*h = Header{}
if len(in) < 4 {
return io.ErrUnexpectedEOF
}
h.HeaderSize += 4
b, in := in[:4], in[4:]
if !bytes.Equal(b, frameMagic) {
if !bytes.Equal(b[1:4], skippableFrameMagic) || b[0]&0xf0 != 0x50 {
return ErrMagicMismatch
}
*h = Header{Skippable: true}
if len(in) < 4 {
return io.ErrUnexpectedEOF
}
h.HeaderSize += 4
h.Skippable = true
h.SkippableID = int(b[0] & 0xf)
h.SkippableSize = binary.LittleEndian.Uint32(in)
return nil
}
if len(in) < 1 {
return io.ErrUnexpectedEOF
}
// Clear output
*h = Header{}
fhd, in := in[0], in[1:]
h.SingleSegment = fhd&(1<<5) != 0
h.HasCheckSum = fhd&(1<<2) != 0
if fhd&(1<<3) != 0 {
return errors.New("reserved bit set on frame header")
}
// Read Window_Descriptor
// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#window_descriptor
if len(in) < 1 {
return io.ErrUnexpectedEOF
}
fhd, in := in[0], in[1:]
h.HeaderSize++
h.SingleSegment = fhd&(1<<5) != 0
h.HasCheckSum = fhd&(1<<2) != 0
if fhd&(1<<3) != 0 {
return errors.New("reserved bit set on frame header")
}
if !h.SingleSegment {
if len(in) < 1 {
return io.ErrUnexpectedEOF
}
var wd byte
wd, in = in[0], in[1:]
h.HeaderSize++
windowLog := 10 + (wd >> 3)
windowBase := uint64(1) << windowLog
windowAdd := (windowBase / 8) * uint64(wd&0x7)
@@ -120,9 +152,7 @@ func (h *Header) Decode(in []byte) error {
return io.ErrUnexpectedEOF
}
b, in = in[:size], in[size:]
if b == nil {
return io.ErrUnexpectedEOF
}
h.HeaderSize += int(size)
switch size {
case 1:
h.DictionaryID = uint32(b[0])
@@ -152,9 +182,7 @@ func (h *Header) Decode(in []byte) error {
return io.ErrUnexpectedEOF
}
b, in = in[:fcsSize], in[fcsSize:]
if b == nil {
return io.ErrUnexpectedEOF
}
h.HeaderSize += int(fcsSize)
switch fcsSize {
case 1:
h.FrameContentSize = uint64(b[0])

24
vendor/github.com/klauspost/compress/zstd/enc_base.go сгенерированный поставляемый
Просмотреть файл

@@ -108,11 +108,6 @@ func (e *fastBase) UseBlock(enc *blockEnc) {
e.blk = enc
}
func (e *fastBase) matchlenNoHist(s, t int32, src []byte) int32 {
// Extend the match to be as long as possible.
return int32(matchLen(src[s:], src[t:]))
}
func (e *fastBase) matchlen(s, t int32, src []byte) int32 {
if debugAsserts {
if s < 0 {
@@ -131,9 +126,24 @@ func (e *fastBase) matchlen(s, t int32, src []byte) int32 {
panic(fmt.Sprintf("len(src)-s (%d) > maxCompressedBlockSize (%d)", len(src)-int(s), maxCompressedBlockSize))
}
}
a := src[s:]
b := src[t:]
b = b[:len(a)]
end := int32((len(a) >> 3) << 3)
for i := int32(0); i < end; i += 8 {
if diff := load6432(a, i) ^ load6432(b, i); diff != 0 {
return i + int32(bits.TrailingZeros64(diff)>>3)
}
}
// Extend the match to be as long as possible.
return int32(matchLen(src[s:], src[t:]))
a = a[end:]
b = b[end:]
for i := range a {
if a[i] != b[i] {
return int32(i) + end
}
}
return int32(len(a)) + end
}
// Reset the encoding table.

139
vendor/github.com/klauspost/compress/zstd/enc_fast.go сгенерированный поставляемый
Просмотреть файл

@@ -6,8 +6,6 @@ package zstd
import (
"fmt"
"math"
"math/bits"
)
const (
@@ -136,20 +134,7 @@ encodeLoop:
// Consider history as well.
var seq seq
var length int32
// length = 4 + e.matchlen(s+6, repIndex+4, src)
{
a := src[s+6:]
b := src[repIndex+4:]
endI := len(a) & (math.MaxInt32 - 7)
length = int32(endI) + 4
for i := 0; i < endI; i += 8 {
if diff := load64(a, i) ^ load64(b, i); diff != 0 {
length = int32(i+bits.TrailingZeros64(diff)>>3) + 4
break
}
}
}
length = 4 + e.matchlen(s+6, repIndex+4, src)
seq.matchLen = uint32(length - zstdMinMatch)
// We might be able to match backwards.
@@ -236,20 +221,7 @@ encodeLoop:
}
// Extend the 4-byte match as long as possible.
//l := e.matchlen(s+4, t+4, src) + 4
var l int32
{
a := src[s+4:]
b := src[t+4:]
endI := len(a) & (math.MaxInt32 - 7)
l = int32(endI) + 4
for i := 0; i < endI; i += 8 {
if diff := load64(a, i) ^ load64(b, i); diff != 0 {
l = int32(i+bits.TrailingZeros64(diff)>>3) + 4
break
}
}
}
l := e.matchlen(s+4, t+4, src) + 4
// Extend backwards
tMin := s - e.maxMatchOff
@@ -286,20 +258,7 @@ encodeLoop:
if o2 := s - offset2; canRepeat && load3232(src, o2) == uint32(cv) {
// We have at least 4 byte match.
// No need to check backwards. We come straight from a match
//l := 4 + e.matchlen(s+4, o2+4, src)
var l int32
{
a := src[s+4:]
b := src[o2+4:]
endI := len(a) & (math.MaxInt32 - 7)
l = int32(endI) + 4
for i := 0; i < endI; i += 8 {
if diff := load64(a, i) ^ load64(b, i); diff != 0 {
l = int32(i+bits.TrailingZeros64(diff)>>3) + 4
break
}
}
}
l := 4 + e.matchlen(s+4, o2+4, src)
// Store this, since we have it.
nextHash := hashLen(cv, hashLog, tableFastHashLen)
@@ -418,21 +377,7 @@ encodeLoop:
if len(blk.sequences) > 2 && load3232(src, repIndex) == uint32(cv>>16) {
// Consider history as well.
var seq seq
// length := 4 + e.matchlen(s+6, repIndex+4, src)
// length := 4 + int32(matchLen(src[s+6:], src[repIndex+4:]))
var length int32
{
a := src[s+6:]
b := src[repIndex+4:]
endI := len(a) & (math.MaxInt32 - 7)
length = int32(endI) + 4
for i := 0; i < endI; i += 8 {
if diff := load64(a, i) ^ load64(b, i); diff != 0 {
length = int32(i+bits.TrailingZeros64(diff)>>3) + 4
break
}
}
}
length := 4 + e.matchlen(s+6, repIndex+4, src)
seq.matchLen = uint32(length - zstdMinMatch)
@@ -522,21 +467,7 @@ encodeLoop:
panic(fmt.Sprintf("t (%d) < 0 ", t))
}
// Extend the 4-byte match as long as possible.
//l := e.matchlenNoHist(s+4, t+4, src) + 4
// l := int32(matchLen(src[s+4:], src[t+4:])) + 4
var l int32
{
a := src[s+4:]
b := src[t+4:]
endI := len(a) & (math.MaxInt32 - 7)
l = int32(endI) + 4
for i := 0; i < endI; i += 8 {
if diff := load64(a, i) ^ load64(b, i); diff != 0 {
l = int32(i+bits.TrailingZeros64(diff)>>3) + 4
break
}
}
}
l := e.matchlen(s+4, t+4, src) + 4
// Extend backwards
tMin := s - e.maxMatchOff
@@ -573,21 +504,7 @@ encodeLoop:
if o2 := s - offset2; len(blk.sequences) > 2 && load3232(src, o2) == uint32(cv) {
// We have at least 4 byte match.
// No need to check backwards. We come straight from a match
//l := 4 + e.matchlenNoHist(s+4, o2+4, src)
// l := 4 + int32(matchLen(src[s+4:], src[o2+4:]))
var l int32
{
a := src[s+4:]
b := src[o2+4:]
endI := len(a) & (math.MaxInt32 - 7)
l = int32(endI) + 4
for i := 0; i < endI; i += 8 {
if diff := load64(a, i) ^ load64(b, i); diff != 0 {
l = int32(i+bits.TrailingZeros64(diff)>>3) + 4
break
}
}
}
l := 4 + e.matchlen(s+4, o2+4, src)
// Store this, since we have it.
nextHash := hashLen(cv, hashLog, tableFastHashLen)
@@ -731,19 +648,7 @@ encodeLoop:
// Consider history as well.
var seq seq
var length int32
// length = 4 + e.matchlen(s+6, repIndex+4, src)
{
a := src[s+6:]
b := src[repIndex+4:]
endI := len(a) & (math.MaxInt32 - 7)
length = int32(endI) + 4
for i := 0; i < endI; i += 8 {
if diff := load64(a, i) ^ load64(b, i); diff != 0 {
length = int32(i+bits.TrailingZeros64(diff)>>3) + 4
break
}
}
}
length = 4 + e.matchlen(s+6, repIndex+4, src)
seq.matchLen = uint32(length - zstdMinMatch)
@@ -831,20 +736,7 @@ encodeLoop:
}
// Extend the 4-byte match as long as possible.
//l := e.matchlen(s+4, t+4, src) + 4
var l int32
{
a := src[s+4:]
b := src[t+4:]
endI := len(a) & (math.MaxInt32 - 7)
l = int32(endI) + 4
for i := 0; i < endI; i += 8 {
if diff := load64(a, i) ^ load64(b, i); diff != 0 {
l = int32(i+bits.TrailingZeros64(diff)>>3) + 4
break
}
}
}
l := e.matchlen(s+4, t+4, src) + 4
// Extend backwards
tMin := s - e.maxMatchOff
@@ -881,20 +773,7 @@ encodeLoop:
if o2 := s - offset2; canRepeat && load3232(src, o2) == uint32(cv) {
// We have at least 4 byte match.
// No need to check backwards. We come straight from a match
//l := 4 + e.matchlen(s+4, o2+4, src)
var l int32
{
a := src[s+4:]
b := src[o2+4:]
endI := len(a) & (math.MaxInt32 - 7)
l = int32(endI) + 4
for i := 0; i < endI; i += 8 {
if diff := load64(a, i) ^ load64(b, i); diff != 0 {
l = int32(i+bits.TrailingZeros64(diff)>>3) + 4
break
}
}
}
l := 4 + e.matchlen(s+4, o2+4, src)
// Store this, since we have it.
nextHash := hashLen(cv, hashLog, tableFastHashLen)

10
vendor/github.com/klauspost/compress/zstd/encoder_options.go сгенерированный поставляемый
Просмотреть файл

@@ -24,6 +24,7 @@ type encoderOptions struct {
allLitEntropy bool
customWindow bool
customALEntropy bool
customBlockSize bool
lowMem bool
dict *dict
}
@@ -33,7 +34,7 @@ func (o *encoderOptions) setDefault() {
concurrent: runtime.GOMAXPROCS(0),
crc: true,
single: nil,
blockSize: 1 << 16,
blockSize: maxCompressedBlockSize,
windowSize: 8 << 20,
level: SpeedDefault,
allLitEntropy: true,
@@ -106,6 +107,7 @@ func WithWindowSize(n int) EOption {
o.customWindow = true
if o.blockSize > o.windowSize {
o.blockSize = o.windowSize
o.customBlockSize = true
}
return nil
}
@@ -188,10 +190,9 @@ func EncoderLevelFromZstd(level int) EncoderLevel {
return SpeedDefault
case level >= 6 && level < 10:
return SpeedBetterCompression
case level >= 10:
default:
return SpeedBestCompression
}
return SpeedDefault
}
// String provides a string representation of the compression level.
@@ -222,6 +223,9 @@ func WithEncoderLevel(l EncoderLevel) EOption {
switch o.level {
case SpeedFastest:
o.windowSize = 4 << 20
if !o.customBlockSize {
o.blockSize = 1 << 16
}
case SpeedDefault:
o.windowSize = 8 << 20
case SpeedBetterCompression:

2
vendor/github.com/klauspost/compress/zstd/fse_decoder.go сгенерированный поставляемый
Просмотреть файл

@@ -379,7 +379,7 @@ func (s decSymbol) final() (int, uint8) {
// This can only be used if no symbols are 0 bits.
// At least tablelog bits must be available in the bit reader.
func (s *fseState) nextFast(br *bitReader) (uint32, uint8) {
lowBits := uint16(br.getBitsFast(s.state.nbBits()))
lowBits := br.get16BitsFast(s.state.nbBits())
s.state = s.dt[s.state.newState()+lowBits]
return s.state.baseline(), s.state.addBits()
}

5
vendor/github.com/klauspost/compress/zstd/fse_encoder.go сгенерированный поставляемый
Просмотреть файл

@@ -62,9 +62,8 @@ func (s symbolTransform) String() string {
// To indicate that you have populated the histogram call HistogramFinished
// with the value of the highest populated symbol, as well as the number of entries
// in the most populated entry. These are accepted at face value.
// The returned slice will always be length 256.
func (s *fseEncoder) Histogram() []uint32 {
return s.count[:]
func (s *fseEncoder) Histogram() *[256]uint32 {
return &s.count
}
// HistogramFinished can be called to indicate that the histogram has been populated.

1
vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s сгенерированный поставляемый
Просмотреть файл

@@ -1,6 +1,7 @@
// +build !appengine
// +build gc
// +build !purego
// +build !noasm
#include "textflag.h"

186
vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,186 @@
// +build gc,!purego,!noasm
#include "textflag.h"
// Register allocation.
#define digest R1
#define h R2 // Return value.
#define p R3 // Input pointer.
#define len R4
#define nblocks R5 // len / 32.
#define prime1 R7
#define prime2 R8
#define prime3 R9
#define prime4 R10
#define prime5 R11
#define v1 R12
#define v2 R13
#define v3 R14
#define v4 R15
#define x1 R20
#define x2 R21
#define x3 R22
#define x4 R23
#define round(acc, x) \
MADD prime2, acc, x, acc \
ROR $64-31, acc \
MUL prime1, acc \
// x = round(0, x).
#define round0(x) \
MUL prime2, x \
ROR $64-31, x \
MUL prime1, x \
#define mergeRound(x) \
round0(x) \
EOR x, h \
MADD h, prime4, prime1, h \
// Update v[1-4] with 32-byte blocks. Assumes len >= 32.
#define blocksLoop() \
LSR $5, len, nblocks \
PCALIGN $16 \
loop: \
LDP.P 32(p), (x1, x2) \
round(v1, x1) \
LDP -16(p), (x3, x4) \
round(v2, x2) \
SUB $1, nblocks \
round(v3, x3) \
round(v4, x4) \
CBNZ nblocks, loop \
// The primes are repeated here to ensure that they're stored
// in a contiguous array, so we can load them with LDP.
DATA primes<> +0(SB)/8, $11400714785074694791
DATA primes<> +8(SB)/8, $14029467366897019727
DATA primes<>+16(SB)/8, $1609587929392839161
DATA primes<>+24(SB)/8, $9650029242287828579
DATA primes<>+32(SB)/8, $2870177450012600261
GLOBL primes<>(SB), NOPTR+RODATA, $40
// func Sum64(b []byte) uint64
TEXT ·Sum64(SB), NOFRAME+NOSPLIT, $0-32
LDP b_base+0(FP), (p, len)
LDP primes<> +0(SB), (prime1, prime2)
LDP primes<>+16(SB), (prime3, prime4)
MOVD primes<>+32(SB), prime5
CMP $32, len
CSEL LO, prime5, ZR, h // if len < 32 { h = prime5 } else { h = 0 }
BLO afterLoop
ADD prime1, prime2, v1
MOVD prime2, v2
MOVD $0, v3
NEG prime1, v4
blocksLoop()
ROR $64-1, v1, x1
ROR $64-7, v2, x2
ADD x1, x2
ROR $64-12, v3, x3
ROR $64-18, v4, x4
ADD x3, x4
ADD x2, x4, h
mergeRound(v1)
mergeRound(v2)
mergeRound(v3)
mergeRound(v4)
afterLoop:
ADD len, h
TBZ $4, len, try8
LDP.P 16(p), (x1, x2)
round0(x1)
ROR $64-27, h
EOR x1 @> 64-27, h, h
MADD h, prime4, prime1, h
round0(x2)
ROR $64-27, h
EOR x2 @> 64-27, h
MADD h, prime4, prime1, h
try8:
TBZ $3, len, try4
MOVD.P 8(p), x1
round0(x1)
ROR $64-27, h
EOR x1 @> 64-27, h
MADD h, prime4, prime1, h
try4:
TBZ $2, len, try2
MOVWU.P 4(p), x2
MUL prime1, x2
ROR $64-23, h
EOR x2 @> 64-23, h
MADD h, prime3, prime2, h
try2:
TBZ $1, len, try1
MOVHU.P 2(p), x3
AND $255, x3, x1
LSR $8, x3, x2
MUL prime5, x1
ROR $64-11, h
EOR x1 @> 64-11, h
MUL prime1, h
MUL prime5, x2
ROR $64-11, h
EOR x2 @> 64-11, h
MUL prime1, h
try1:
TBZ $0, len, end
MOVBU (p), x4
MUL prime5, x4
ROR $64-11, h
EOR x4 @> 64-11, h
MUL prime1, h
end:
EOR h >> 33, h
MUL prime2, h
EOR h >> 29, h
MUL prime3, h
EOR h >> 32, h
MOVD h, ret+24(FP)
RET
// func writeBlocks(d *Digest, b []byte) int
//
// Assumes len(b) >= 32.
TEXT ·writeBlocks(SB), NOFRAME+NOSPLIT, $0-40
LDP primes<>(SB), (prime1, prime2)
// Load state. Assume v[1-4] are stored contiguously.
MOVD d+0(FP), digest
LDP 0(digest), (v1, v2)
LDP 16(digest), (v3, v4)
LDP b_base+8(FP), (p, len)
blocksLoop()
// Store updated state.
STP (v1, v2), 0(digest)
STP (v3, v4), 16(digest)
BIC $31, len
MOVD len, ret+32(FP)
RET

8
vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.go → vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_asm.go сгенерированный поставляемый
Просмотреть файл

@@ -1,5 +1,9 @@
//go:build !appengine && gc && !purego
// +build !appengine,gc,!purego
//go:build (amd64 || arm64) && !appengine && gc && !purego && !noasm
// +build amd64 arm64
// +build !appengine
// +build gc
// +build !purego
// +build !noasm
package xxhash

4
vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go сгенерированный поставляемый
Просмотреть файл

@@ -1,5 +1,5 @@
//go:build !amd64 || appengine || !gc || purego
// +build !amd64 appengine !gc purego
//go:build (!amd64 && !arm64) || appengine || !gc || purego || noasm
// +build !amd64,!arm64 appengine !gc purego noasm
package xxhash

4
vendor/github.com/klauspost/compress/zstd/seqdec.go сгенерированный поставляемый
Просмотреть файл

@@ -278,7 +278,7 @@ func (s *sequenceDecs) decode(seqs int, br *bitReader, hist []byte) error {
mlState = mlTable[mlState.newState()&maxTableMask]
ofState = ofTable[ofState.newState()&maxTableMask]
} else {
bits := br.getBitsFast(nBits)
bits := br.get32BitsFast(nBits)
lowBits := uint16(bits >> ((ofState.nbBits() + mlState.nbBits()) & 31))
llState = llTable[(llState.newState()+lowBits)&maxTableMask]
@@ -326,7 +326,7 @@ func (s *sequenceDecs) updateAlt(br *bitReader) {
s.offsets.state.state = s.offsets.state.dt[c.newState()]
return
}
bits := br.getBitsFast(nBits)
bits := br.get32BitsFast(nBits)
lowBits := uint16(bits >> ((c.nbBits() + b.nbBits()) & 31))
s.litLengths.state.state = s.litLengths.state.dt[a.newState()+lowBits]

4
vendor/github.com/klauspost/cpuid/v2/README.md сгенерированный поставляемый
Просмотреть файл

@@ -39,10 +39,10 @@ func main() {
fmt.Println("ThreadsPerCore:", CPU.ThreadsPerCore)
fmt.Println("LogicalCores:", CPU.LogicalCores)
fmt.Println("Family", CPU.Family, "Model:", CPU.Model, "Vendor ID:", CPU.VendorID)
fmt.Println("Features:", fmt.Sprintf(strings.Join(CPU.FeatureSet(), ",")))
fmt.Println("Features:", strings.Join(CPU.FeatureSet(), ","))
fmt.Println("Cacheline bytes:", CPU.CacheLine)
fmt.Println("L1 Data Cache:", CPU.Cache.L1D, "bytes")
fmt.Println("L1 Instruction Cache:", CPU.Cache.L1D, "bytes")
fmt.Println("L1 Instruction Cache:", CPU.Cache.L1I, "bytes")
fmt.Println("L2 Cache:", CPU.Cache.L2, "bytes")
fmt.Println("L3 Cache:", CPU.Cache.L3, "bytes")
fmt.Println("Frequency", CPU.Hz, "hz")

74
vendor/github.com/klauspost/cpuid/v2/cpuid.go сгенерированный поставляемый
Просмотреть файл

@@ -95,10 +95,13 @@ const (
AVXSLOW // Indicates the CPU performs 2 128 bit operations instead of one.
BMI1 // Bit Manipulation Instruction Set 1
BMI2 // Bit Manipulation Instruction Set 2
CETIBT // Intel CET Indirect Branch Tracking
CETSS // Intel CET Shadow Stack
CLDEMOTE // Cache Line Demote
CLMUL // Carry-less Multiplication
CLZERO // CLZERO instruction supported
CMOV // i686 CMOV
CMPXCHG8 // CMPXCHG8 instruction
CPBOOST // Core Performance Boost
CX16 // CMPXCHG16B Instruction
ENQCMD // Enqueue Command
@@ -106,6 +109,8 @@ const (
F16C // Half-precision floating-point conversion
FMA3 // Intel FMA 3. Does not imply AVX.
FMA4 // Bulldozer FMA4 functions
FXSR // FXSAVE, FXRESTOR instructions, CR4 bit 9
FXSROPT // FXSAVE/FXRSTOR optimizations
GFNI // Galois Field New Instructions
HLE // Hardware Lock Elision
HTT // Hyperthreading (enabled)
@@ -123,16 +128,19 @@ const (
IBSRIPINVALIDCHK // Instruction Based Sampling Feature (AMD)
INT_WBINVD // WBINVD/WBNOINVD are interruptible.
INVLPGB // NVLPGB and TLBSYNC instruction supported
LAHF // LAHF/SAHF in long mode
LZCNT // LZCNT instruction
MCAOVERFLOW // MCA overflow recovery support.
MCOMMIT // MCOMMIT instruction supported
MMX // standard MMX
MMXEXT // SSE integer functions or AMD MMX ext
MOVBE // MOVBE instruction (big-endian)
MOVDIR64B // Move 64 Bytes as Direct Store
MOVDIRI // Move Doubleword as Direct Store
MPX // Intel MPX (Memory Protection Extensions)
MSRIRC // Instruction Retired Counter MSR available
NX // NX (No-Execute) bit
OSXSAVE // XSAVE enabled by OS
POPCNT // POPCNT instruction
RDPRU // RDPRU instruction supported
RDRAND // RDRAND instruction is available
@@ -140,6 +148,7 @@ const (
RDTSCP // RDTSCP Instruction
RTM // Restricted Transactional Memory
RTM_ALWAYS_ABORT // Indicates that the loaded microcode is forcing RTM abort.
SCE // SYSENTER and SYSEXIT instructions
SERIALIZE // Serialize Instruction Execution
SGX // Software Guard Extensions
SGXLC // Software Guard Extensions Launch Control
@@ -160,7 +169,9 @@ const (
VPCLMULQDQ // Carry-Less Multiplication Quadword
WAITPKG // TPAUSE, UMONITOR, UMWAIT
WBNOINVD // Write Back and Do Not Invalidate Cache
X87 // FPU
XOP // Bulldozer XOP functions
XSAVE // XSAVE, XRESTOR, XSETBV, XGETBV
// ARM features:
AESARM // AES instructions
@@ -311,6 +322,31 @@ func (c CPUInfo) Has(id FeatureID) bool {
return c.featureSet.inSet(id)
}
// https://en.wikipedia.org/wiki/X86-64#Microarchitecture_levels
var level1Features = flagSetWith(CMOV, CMPXCHG8, X87, FXSR, MMX, SCE, SSE, SSE2)
var level2Features = flagSetWith(CMOV, CMPXCHG8, X87, FXSR, MMX, SCE, SSE, SSE2, CX16, LAHF, POPCNT, SSE3, SSE4, SSE42, SSSE3)
var level3Features = flagSetWith(CMOV, CMPXCHG8, X87, FXSR, MMX, SCE, SSE, SSE2, CX16, LAHF, POPCNT, SSE3, SSE4, SSE42, SSSE3, AVX, AVX2, BMI1, BMI2, F16C, FMA3, LZCNT, MOVBE, OSXSAVE)
var level4Features = flagSetWith(CMOV, CMPXCHG8, X87, FXSR, MMX, SCE, SSE, SSE2, CX16, LAHF, POPCNT, SSE3, SSE4, SSE42, SSSE3, AVX, AVX2, BMI1, BMI2, F16C, FMA3, LZCNT, MOVBE, OSXSAVE, AVX512F, AVX512BW, AVX512CD, AVX512DQ, AVX512VL)
// X64Level returns the microarchitecture level detected on the CPU.
// If features are lacking or non x64 mode, 0 is returned.
// See https://en.wikipedia.org/wiki/X86-64#Microarchitecture_levels
func (c CPUInfo) X64Level() int {
if c.featureSet.hasSet(level4Features) {
return 4
}
if c.featureSet.hasSet(level3Features) {
return 3
}
if c.featureSet.hasSet(level2Features) {
return 2
}
if c.featureSet.hasSet(level1Features) {
return 1
}
return 0
}
// Disable will disable one or several features.
func (c *CPUInfo) Disable(ids ...FeatureID) bool {
for _, id := range ids {
@@ -335,9 +371,7 @@ func (c CPUInfo) IsVendor(v Vendor) bool {
func (c CPUInfo) FeatureSet() []string {
s := make([]string, 0)
for _, f := range c.featureSet.Strings() {
s = append(s, f)
}
s = append(s, c.featureSet.Strings()...)
return s
}
@@ -499,6 +533,24 @@ func (s *flagSet) or(other flagSet) {
}
}
// hasSet returns whether all features are present.
func (s flagSet) hasSet(other flagSet) bool {
for i, v := range other[:] {
if s[i]&v != v {
return false
}
}
return true
}
func flagSetWith(feat ...FeatureID) flagSet {
var res flagSet
for _, f := range feat {
res.set(f)
}
return res
}
// ParseFeature will parse the string and return the ID of the matching feature.
// Will return UNKNOWN if not found.
func ParseFeature(s string) FeatureID {
@@ -708,6 +760,7 @@ func (c *CPUInfo) cacheSize() {
if maxFunctionID() < 4 {
return
}
c.Cache.L1I, c.Cache.L1D, c.Cache.L2, c.Cache.L3 = 0, 0, 0, 0
for i := uint32(0); ; i++ {
eax, ebx, ecx, _ := cpuidex(4, i)
cacheType := eax & 15
@@ -800,8 +853,6 @@ func (c *CPUInfo) cacheSize() {
}
}
}
return
}
type SGXEPCSection struct {
@@ -865,9 +916,14 @@ func support() flagSet {
family, model := familyModel()
_, _, c, d := cpuid(1)
fs.setIf((d&(1<<0)) != 0, X87)
fs.setIf((d&(1<<8)) != 0, CMPXCHG8)
fs.setIf((d&(1<<11)) != 0, SCE)
fs.setIf((d&(1<<15)) != 0, CMOV)
fs.setIf((d&(1<<22)) != 0, MMXEXT)
fs.setIf((d&(1<<23)) != 0, MMX)
fs.setIf((d&(1<<25)) != 0, MMXEXT)
fs.setIf((d&(1<<24)) != 0, FXSR)
fs.setIf((d&(1<<25)) != 0, FXSROPT)
fs.setIf((d&(1<<25)) != 0, SSE)
fs.setIf((d&(1<<26)) != 0, SSE2)
fs.setIf((c&1) != 0, SSE3)
@@ -877,6 +933,7 @@ func support() flagSet {
fs.setIf((c&0x00100000) != 0, SSE42)
fs.setIf((c&(1<<25)) != 0, AESNI)
fs.setIf((c&(1<<1)) != 0, CLMUL)
fs.setIf(c&(1<<22) != 0, MOVBE)
fs.setIf(c&(1<<23) != 0, POPCNT)
fs.setIf(c&(1<<30) != 0, RDRAND)
@@ -892,6 +949,8 @@ func support() flagSet {
if vend == AMD && (d&(1<<28)) != 0 && mfi >= 4 {
fs.setIf(threadsPerCore() > 1, HTT)
}
fs.setIf(c&1<<26 != 0, XSAVE)
fs.setIf(c&1<<27 != 0, OSXSAVE)
// Check XGETBV/XSAVE (26), OXSAVE (27) and AVX (28) bits
const avxCheck = 1<<26 | 1<<27 | 1<<28
if c&avxCheck == avxCheck {
@@ -936,6 +995,7 @@ func support() flagSet {
fs.setIf(ebx&(1<<29) != 0, SHA)
// CPUID.(EAX=7, ECX=0).ECX
fs.setIf(ecx&(1<<5) != 0, WAITPKG)
fs.setIf(ecx&(1<<7) != 0, CETSS)
fs.setIf(ecx&(1<<25) != 0, CLDEMOTE)
fs.setIf(ecx&(1<<27) != 0, MOVDIRI)
fs.setIf(ecx&(1<<28) != 0, MOVDIR64B)
@@ -945,6 +1005,7 @@ func support() flagSet {
fs.setIf(edx&(1<<11) != 0, RTM_ALWAYS_ABORT)
fs.setIf(edx&(1<<14) != 0, SERIALIZE)
fs.setIf(edx&(1<<16) != 0, TSXLDTRK)
fs.setIf(edx&(1<<20) != 0, CETIBT)
fs.setIf(edx&(1<<26) != 0, IBPB)
fs.setIf(edx&(1<<27) != 0, STIBP)
@@ -996,6 +1057,7 @@ func support() flagSet {
fs.set(LZCNT)
fs.set(POPCNT)
}
fs.setIf((c&(1<<0)) != 0, LAHF)
fs.setIf((c&(1<<10)) != 0, IBS)
fs.setIf((d&(1<<31)) != 0, AMD3DNOW)
fs.setIf((d&(1<<30)) != 0, AMD3DNOWEXT)

3
vendor/github.com/klauspost/cpuid/v2/detect_arm64.go сгенерированный поставляемый
Просмотреть файл

@@ -1,6 +1,7 @@
// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file.
//+build arm64,!gccgo,!noasm,!appengine
//go:build arm64 && !gccgo && !noasm && !appengine
// +build arm64,!gccgo,!noasm,!appengine
package cpuid

3
vendor/github.com/klauspost/cpuid/v2/detect_ref.go сгенерированный поставляемый
Просмотреть файл

@@ -1,6 +1,7 @@
// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file.
//+build !amd64,!386,!arm64 gccgo noasm appengine
//go:build (!amd64 && !386 && !arm64) || gccgo || noasm || appengine
// +build !amd64,!386,!arm64 gccgo noasm appengine
package cpuid

3
vendor/github.com/klauspost/cpuid/v2/detect_x86.go сгенерированный поставляемый
Просмотреть файл

@@ -1,6 +1,7 @@
// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file.
//+build 386,!gccgo,!noasm,!appengine amd64,!gccgo,!noasm,!appengine
//go:build (386 && !gccgo && !noasm && !appengine) || (amd64 && !gccgo && !noasm && !appengine)
// +build 386,!gccgo,!noasm,!appengine amd64,!gccgo,!noasm,!appengine
package cpuid

197
vendor/github.com/klauspost/cpuid/v2/featureid_string.go сгенерированный поставляемый
Просмотреть файл

@@ -36,103 +36,114 @@ func _() {
_ = x[AVXSLOW-26]
_ = x[BMI1-27]
_ = x[BMI2-28]
_ = x[CLDEMOTE-29]
_ = x[CLMUL-30]
_ = x[CLZERO-31]
_ = x[CMOV-32]
_ = x[CPBOOST-33]
_ = x[CX16-34]
_ = x[ENQCMD-35]
_ = x[ERMS-36]
_ = x[F16C-37]
_ = x[FMA3-38]
_ = x[FMA4-39]
_ = x[GFNI-40]
_ = x[HLE-41]
_ = x[HTT-42]
_ = x[HWA-43]
_ = x[HYPERVISOR-44]
_ = x[IBPB-45]
_ = x[IBS-46]
_ = x[IBSBRNTRGT-47]
_ = x[IBSFETCHSAM-48]
_ = x[IBSFFV-49]
_ = x[IBSOPCNT-50]
_ = x[IBSOPCNTEXT-51]
_ = x[IBSOPSAM-52]
_ = x[IBSRDWROPCNT-53]
_ = x[IBSRIPINVALIDCHK-54]
_ = x[INT_WBINVD-55]
_ = x[INVLPGB-56]
_ = x[LZCNT-57]
_ = x[MCAOVERFLOW-58]
_ = x[MCOMMIT-59]
_ = x[MMX-60]
_ = x[MMXEXT-61]
_ = x[MOVDIR64B-62]
_ = x[MOVDIRI-63]
_ = x[MPX-64]
_ = x[MSRIRC-65]
_ = x[NX-66]
_ = x[POPCNT-67]
_ = x[RDPRU-68]
_ = x[RDRAND-69]
_ = x[RDSEED-70]
_ = x[RDTSCP-71]
_ = x[RTM-72]
_ = x[RTM_ALWAYS_ABORT-73]
_ = x[SERIALIZE-74]
_ = x[SGX-75]
_ = x[SGXLC-76]
_ = x[SHA-77]
_ = x[SSE-78]
_ = x[SSE2-79]
_ = x[SSE3-80]
_ = x[SSE4-81]
_ = x[SSE42-82]
_ = x[SSE4A-83]
_ = x[SSSE3-84]
_ = x[STIBP-85]
_ = x[SUCCOR-86]
_ = x[TBM-87]
_ = x[TSXLDTRK-88]
_ = x[VAES-89]
_ = x[VMX-90]
_ = x[VPCLMULQDQ-91]
_ = x[WAITPKG-92]
_ = x[WBNOINVD-93]
_ = x[XOP-94]
_ = x[AESARM-95]
_ = x[ARMCPUID-96]
_ = x[ASIMD-97]
_ = x[ASIMDDP-98]
_ = x[ASIMDHP-99]
_ = x[ASIMDRDM-100]
_ = x[ATOMICS-101]
_ = x[CRC32-102]
_ = x[DCPOP-103]
_ = x[EVTSTRM-104]
_ = x[FCMA-105]
_ = x[FP-106]
_ = x[FPHP-107]
_ = x[GPA-108]
_ = x[JSCVT-109]
_ = x[LRCPC-110]
_ = x[PMULL-111]
_ = x[SHA1-112]
_ = x[SHA2-113]
_ = x[SHA3-114]
_ = x[SHA512-115]
_ = x[SM3-116]
_ = x[SM4-117]
_ = x[SVE-118]
_ = x[lastID-119]
_ = x[CETIBT-29]
_ = x[CETSS-30]
_ = x[CLDEMOTE-31]
_ = x[CLMUL-32]
_ = x[CLZERO-33]
_ = x[CMOV-34]
_ = x[CMPXCHG8-35]
_ = x[CPBOOST-36]
_ = x[CX16-37]
_ = x[ENQCMD-38]
_ = x[ERMS-39]
_ = x[F16C-40]
_ = x[FMA3-41]
_ = x[FMA4-42]
_ = x[FXSR-43]
_ = x[FXSROPT-44]
_ = x[GFNI-45]
_ = x[HLE-46]
_ = x[HTT-47]
_ = x[HWA-48]
_ = x[HYPERVISOR-49]
_ = x[IBPB-50]
_ = x[IBS-51]
_ = x[IBSBRNTRGT-52]
_ = x[IBSFETCHSAM-53]
_ = x[IBSFFV-54]
_ = x[IBSOPCNT-55]
_ = x[IBSOPCNTEXT-56]
_ = x[IBSOPSAM-57]
_ = x[IBSRDWROPCNT-58]
_ = x[IBSRIPINVALIDCHK-59]
_ = x[INT_WBINVD-60]
_ = x[INVLPGB-61]
_ = x[LAHF-62]
_ = x[LZCNT-63]
_ = x[MCAOVERFLOW-64]
_ = x[MCOMMIT-65]
_ = x[MMX-66]
_ = x[MMXEXT-67]
_ = x[MOVBE-68]
_ = x[MOVDIR64B-69]
_ = x[MOVDIRI-70]
_ = x[MPX-71]
_ = x[MSRIRC-72]
_ = x[NX-73]
_ = x[OSXSAVE-74]
_ = x[POPCNT-75]
_ = x[RDPRU-76]
_ = x[RDRAND-77]
_ = x[RDSEED-78]
_ = x[RDTSCP-79]
_ = x[RTM-80]
_ = x[RTM_ALWAYS_ABORT-81]
_ = x[SCE-82]
_ = x[SERIALIZE-83]
_ = x[SGX-84]
_ = x[SGXLC-85]
_ = x[SHA-86]
_ = x[SSE-87]
_ = x[SSE2-88]
_ = x[SSE3-89]
_ = x[SSE4-90]
_ = x[SSE42-91]
_ = x[SSE4A-92]
_ = x[SSSE3-93]
_ = x[STIBP-94]
_ = x[SUCCOR-95]
_ = x[TBM-96]
_ = x[TSXLDTRK-97]
_ = x[VAES-98]
_ = x[VMX-99]
_ = x[VPCLMULQDQ-100]
_ = x[WAITPKG-101]
_ = x[WBNOINVD-102]
_ = x[X87-103]
_ = x[XOP-104]
_ = x[XSAVE-105]
_ = x[AESARM-106]
_ = x[ARMCPUID-107]
_ = x[ASIMD-108]
_ = x[ASIMDDP-109]
_ = x[ASIMDHP-110]
_ = x[ASIMDRDM-111]
_ = x[ATOMICS-112]
_ = x[CRC32-113]
_ = x[DCPOP-114]
_ = x[EVTSTRM-115]
_ = x[FCMA-116]
_ = x[FP-117]
_ = x[FPHP-118]
_ = x[GPA-119]
_ = x[JSCVT-120]
_ = x[LRCPC-121]
_ = x[PMULL-122]
_ = x[SHA1-123]
_ = x[SHA2-124]
_ = x[SHA3-125]
_ = x[SHA512-126]
_ = x[SM3-127]
_ = x[SM4-128]
_ = x[SVE-129]
_ = x[lastID-130]
_ = x[firstID-0]
}
const _FeatureID_name = "firstIDADXAESNIAMD3DNOWAMD3DNOWEXTAMXBF16AMXINT8AMXTILEAVXAVX2AVX512BF16AVX512BITALGAVX512BWAVX512CDAVX512DQAVX512ERAVX512FAVX512FP16AVX512IFMAAVX512PFAVX512VBMIAVX512VBMI2AVX512VLAVX512VNNIAVX512VP2INTERSECTAVX512VPOPCNTDQAVXSLOWBMI1BMI2CLDEMOTECLMULCLZEROCMOVCPBOOSTCX16ENQCMDERMSF16CFMA3FMA4GFNIHLEHTTHWAHYPERVISORIBPBIBSIBSBRNTRGTIBSFETCHSAMIBSFFVIBSOPCNTIBSOPCNTEXTIBSOPSAMIBSRDWROPCNTIBSRIPINVALIDCHKINT_WBINVDINVLPGBLZCNTMCAOVERFLOWMCOMMITMMXMMXEXTMOVDIR64BMOVDIRIMPXMSRIRCNXPOPCNTRDPRURDRANDRDSEEDRDTSCPRTMRTM_ALWAYS_ABORTSERIALIZESGXSGXLCSHASSESSE2SSE3SSE4SSE42SSE4ASSSE3STIBPSUCCORTBMTSXLDTRKVAESVMXVPCLMULQDQWAITPKGWBNOINVDXOPAESARMARMCPUIDASIMDASIMDDPASIMDHPASIMDRDMATOMICSCRC32DCPOPEVTSTRMFCMAFPFPHPGPAJSCVTLRCPCPMULLSHA1SHA2SHA3SHA512SM3SM4SVElastID"
const _FeatureID_name = "firstIDADXAESNIAMD3DNOWAMD3DNOWEXTAMXBF16AMXINT8AMXTILEAVXAVX2AVX512BF16AVX512BITALGAVX512BWAVX512CDAVX512DQAVX512ERAVX512FAVX512FP16AVX512IFMAAVX512PFAVX512VBMIAVX512VBMI2AVX512VLAVX512VNNIAVX512VP2INTERSECTAVX512VPOPCNTDQAVXSLOWBMI1BMI2CETIBTCETSSCLDEMOTECLMULCLZEROCMOVCMPXCHG8CPBOOSTCX16ENQCMDERMSF16CFMA3FMA4FXSRFXSROPTGFNIHLEHTTHWAHYPERVISORIBPBIBSIBSBRNTRGTIBSFETCHSAMIBSFFVIBSOPCNTIBSOPCNTEXTIBSOPSAMIBSRDWROPCNTIBSRIPINVALIDCHKINT_WBINVDINVLPGBLAHFLZCNTMCAOVERFLOWMCOMMITMMXMMXEXTMOVBEMOVDIR64BMOVDIRIMPXMSRIRCNXOSXSAVEPOPCNTRDPRURDRANDRDSEEDRDTSCPRTMRTM_ALWAYS_ABORTSCESERIALIZESGXSGXLCSHASSESSE2SSE3SSE4SSE42SSE4ASSSE3STIBPSUCCORTBMTSXLDTRKVAESVMXVPCLMULQDQWAITPKGWBNOINVDX87XOPXSAVEAESARMARMCPUIDASIMDASIMDDPASIMDHPASIMDRDMATOMICSCRC32DCPOPEVTSTRMFCMAFPFPHPGPAJSCVTLRCPCPMULLSHA1SHA2SHA3SHA512SM3SM4SVElastID"
var _FeatureID_index = [...]uint16{0, 7, 10, 15, 23, 34, 41, 48, 55, 58, 62, 72, 84, 92, 100, 108, 116, 123, 133, 143, 151, 161, 172, 180, 190, 208, 223, 230, 234, 238, 246, 251, 257, 261, 268, 272, 278, 282, 286, 290, 294, 298, 301, 304, 307, 317, 321, 324, 334, 345, 351, 359, 370, 378, 390, 406, 416, 423, 428, 439, 446, 449, 455, 464, 471, 474, 480, 482, 488, 493, 499, 505, 511, 514, 530, 539, 542, 547, 550, 553, 557, 561, 565, 570, 575, 580, 585, 591, 594, 602, 606, 609, 619, 626, 634, 637, 643, 651, 656, 663, 670, 678, 685, 690, 695, 702, 706, 708, 712, 715, 720, 725, 730, 734, 738, 742, 748, 751, 754, 757, 763}
var _FeatureID_index = [...]uint16{0, 7, 10, 15, 23, 34, 41, 48, 55, 58, 62, 72, 84, 92, 100, 108, 116, 123, 133, 143, 151, 161, 172, 180, 190, 208, 223, 230, 234, 238, 244, 249, 257, 262, 268, 272, 280, 287, 291, 297, 301, 305, 309, 313, 317, 324, 328, 331, 334, 337, 347, 351, 354, 364, 375, 381, 389, 400, 408, 420, 436, 446, 453, 457, 462, 473, 480, 483, 489, 494, 503, 510, 513, 519, 521, 528, 534, 539, 545, 551, 557, 560, 576, 579, 588, 591, 596, 599, 602, 606, 610, 614, 619, 624, 629, 634, 640, 643, 651, 655, 658, 668, 675, 683, 686, 689, 694, 700, 708, 713, 720, 727, 735, 742, 747, 752, 759, 763, 765, 769, 772, 777, 782, 787, 791, 795, 799, 805, 808, 811, 814, 820}
func (i FeatureID) String() string {
if i < 0 || i >= FeatureID(len(_FeatureID_index)-1) {

2
vendor/github.com/klauspost/cpuid/v2/go.mod сгенерированный поставляемый
Просмотреть файл

@@ -1,3 +1,3 @@
module github.com/klauspost/cpuid/v2
go 1.13
go 1.15

5
vendor/github.com/klauspost/cpuid/v2/os_other_arm64.go сгенерированный поставляемый
Просмотреть файл

@@ -1,8 +1,7 @@
// Copyright (c) 2020 Klaus Post, released under MIT License. See LICENSE file.
// +build arm64
// +build !linux
// +build !darwin
//go:build arm64 && !linux && !darwin
// +build arm64,!linux,!darwin
package cpuid

3
vendor/github.com/klauspost/cpuid/v2/os_safe_linux_arm64.go сгенерированный поставляемый
Просмотреть файл

@@ -1,6 +1,7 @@
// Copyright (c) 2021 Klaus Post, released under MIT License. See LICENSE file.
//+build nounsafe
//go:build nounsafe
// +build nounsafe
package cpuid

3
vendor/github.com/klauspost/cpuid/v2/os_unsafe_linux_arm64.go сгенерированный поставляемый
Просмотреть файл

@@ -1,6 +1,7 @@
// Copyright (c) 2021 Klaus Post, released under MIT License. See LICENSE file.
//+build !nounsafe
//go:build !nounsafe
// +build !nounsafe
package cpuid

4
vendor/github.com/microcosm-cc/bluemonday/.editorconfig сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,4 @@
root = true
[*]
end_of_line = lf

1
vendor/github.com/microcosm-cc/bluemonday/.gitattributes сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1 @@
* text=auto eol=lf

15
vendor/github.com/microcosm-cc/bluemonday/.gitignore сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,15 @@
# Binaries for programs and plugins
*.exe
*.exe~
*.dll
*.so
*.dylib
# Test binary, built with `go test -c`
*.test
# Output of the go coverage tool, specifically when used with LiteIDE
*.out
# goland idea folder
*.idea

6
vendor/github.com/microcosm-cc/bluemonday/.travis.yml сгенерированный поставляемый
Просмотреть файл

@@ -1,6 +1,5 @@
language: go
go:
- 1.1.x
- 1.2.x
- 1.3.x
- 1.4.x
@@ -11,6 +10,11 @@ go:
- 1.9.x
- 1.10.x
- 1.11.x
- 1.12.x
- 1.13.x
- 1.14.x
- 1.15.x
- 1.16.x
- tip
matrix:
allow_failures:

1
vendor/github.com/microcosm-cc/bluemonday/CONTRIBUTING.md сгенерированный поставляемый
Просмотреть файл

@@ -9,6 +9,7 @@ Third-party patches are essential for keeping bluemonday secure and offering the
## Guidelines
1. Do not vendor dependencies. As a security package, were we to vendor dependencies the projects that then vendor bluemonday may not receive the latest security updates to the dependencies. By not vendoring dependencies the project that implements bluemonday will vendor the latest version of any dependent packages. Vendoring is a project problem, not a package problem. bluemonday will be tested against the latest version of dependencies periodically and during any PR/merge.
2. I do not care about spelling mistakes or whitespace and I do not believe that you should either. PRs therefore must be functional in their nature or be substantial and impactful if documentation or examples.
## Submitting an Issue

8
vendor/github.com/microcosm-cc/bluemonday/CREDITS.md сгенерированный поставляемый
Просмотреть файл

@@ -1,6 +1,8 @@
1. Andrew Krasichkov @buglloc https://github.com/buglloc
1. John Graham-Cumming http://jgc.org/
1. Mohammad Gufran https://github.com/Gufran
1. Steven Gutzwiller https://github.com/StevenGutzwiller
1. Andrew Krasichkov @buglloc https://github.com/buglloc
1. Mike Samuel mikesamuel@gmail.com
1. Dmitri Shuralyov shurcooL@gmail.com
1. https://github.com/opennota
1. https://github.com/Gufran
1. opennota https://github.com/opennota https://gitlab.com/opennota
1. Tom Anthony https://www.tomanthony.co.uk/

10
vendor/github.com/microcosm-cc/bluemonday/Makefile сгенерированный поставляемый
Просмотреть файл

@@ -3,6 +3,7 @@
# all: Builds the code locally after testing
#
# fmt: Formats the source files
# fmt-check: Check if the source files are formated
# build: Builds the code locally
# vet: Vets the code
# lint: Runs lint over the code (you do not need to fix everything)
@@ -11,6 +12,8 @@
#
# install: Builds, tests and installs the code locally
GOFILES_NOVENDOR = $(shell find . -type f -name '*.go' -not -path "./vendor/*" -not -path "./.git/*")
.PHONY: all fmt build vet lint test cover install
# The first target is always the default action if `make` is called without
@@ -19,13 +22,16 @@
all: fmt vet test install
fmt:
@gofmt -s -w ./$*
@gofmt -s -w ${GOFILES_NOVENDOR}
fmt-check:
@([ -z "$(shell gofmt -d $(GOFILES_NOVENDOR) | head)" ]) || (echo "Source is unformatted"; exit 1)
build:
@go build
vet:
@go vet *.go
@go vet
lint:
@golint *.go

98
vendor/github.com/microcosm-cc/bluemonday/README.md сгенерированный поставляемый
Просмотреть файл

@@ -2,7 +2,7 @@
bluemonday is a HTML sanitizer implemented in Go. It is fast and highly configurable.
bluemonday takes untrusted user generated content as an input, and will return HTML that has been sanitised against a whitelist of approved HTML elements and attributes so that you can safely include the content in your web page.
bluemonday takes untrusted user generated content as an input, and will return HTML that has been sanitised against an allowlist of approved HTML elements and attributes so that you can safely include the content in your web page.
If you accept user generated content, and your server uses Go, you **need** bluemonday.
@@ -50,18 +50,20 @@ bluemonday is heavily inspired by both the [OWASP Java HTML Sanitizer](https://c
## Technical Summary
Whitelist based, you need to either build a policy describing the HTML elements and attributes to permit (and the `regexp` patterns of attributes), or use one of the supplied policies representing good defaults.
Allowlist based, you need to either build a policy describing the HTML elements and attributes to permit (and the `regexp` patterns of attributes), or use one of the supplied policies representing good defaults.
The policy containing the whitelist is applied using a fast non-validating, forward only, token-based parser implemented in the [Go net/html library](https://godoc.org/golang.org/x/net/html) by the core Go team.
The policy containing the allowlist is applied using a fast non-validating, forward only, token-based parser implemented in the [Go net/html library](https://godoc.org/golang.org/x/net/html) by the core Go team.
We expect to be supplied with well-formatted HTML (closing elements for every applicable open element, nested correctly) and so we do not focus on repairing badly nested or incomplete HTML. We focus on simply ensuring that whatever elements do exist are described in the policy whitelist and that attributes and links are safe for use on your web page. [GIGO](http://en.wikipedia.org/wiki/Garbage_in,_garbage_out) does apply and if you feed it bad HTML bluemonday is not tasked with figuring out how to make it good again.
We expect to be supplied with well-formatted HTML (closing elements for every applicable open element, nested correctly) and so we do not focus on repairing badly nested or incomplete HTML. We focus on simply ensuring that whatever elements do exist are described in the policy allowlist and that attributes and links are safe for use on your web page. [GIGO](http://en.wikipedia.org/wiki/Garbage_in,_garbage_out) does apply and if you feed it bad HTML bluemonday is not tasked with figuring out how to make it good again.
### Supported Go Versions
bluemonday is tested against Go 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, and tip.
bluemonday is tested on all versions since Go 1.2 including tip.
We do not support Go 1.0 as we depend on `golang.org/x/net/html` which includes a reference to `io.ErrNoProgress` which did not exist in Go 1.0.
We support Go 1.1 but Travis no longer tests against it.
## Is it production ready?
*Yes*
@@ -90,7 +92,7 @@ func main() {
// Do this once for each unique policy, and use the policy for the life of the program
// Policy creation/editing is not safe to use in multiple goroutines
p := bluemonday.UGCPolicy()
// The policy can then be used to sanitize lots of input and it is safe to use the policy in multiple goroutines
html := p.Sanitize(
`<a onblur="alert(secret)" href="http://www.google.com">Google</a>`,
@@ -144,8 +146,8 @@ func main() {
We ship two default policies:
1. `bluemonday.StrictPolicy()` which can be thought of as equivalent to stripping all HTML elements and their attributes as it has nothing on its whitelist. An example usage scenario would be blog post titles where HTML tags are not expected at all and if they are then the elements *and* the content of the elements should be stripped. This is a *very* strict policy.
2. `bluemonday.UGCPolicy()` which allows a broad selection of HTML elements and attributes that are safe for user generated content. Note that this policy does *not* whitelist iframes, object, embed, styles, script, etc. An example usage scenario would be blog post bodies where a variety of formatting is expected along with the potential for TABLEs and IMGs.
1. `bluemonday.StrictPolicy()` which can be thought of as equivalent to stripping all HTML elements and their attributes as it has nothing on its allowlist. An example usage scenario would be blog post titles where HTML tags are not expected at all and if they are then the elements *and* the content of the elements should be stripped. This is a *very* strict policy.
2. `bluemonday.UGCPolicy()` which allows a broad selection of HTML elements and attributes that are safe for user generated content. Note that this policy does *not* allow iframes, object, embed, styles, script, etc. An example usage scenario would be blog post bodies where a variety of formatting is expected along with the potential for TABLEs and IMGs.
## Policy Building
@@ -167,12 +169,26 @@ To add elements to a policy either add just the elements:
p.AllowElements("b", "strong")
```
Or using a regex:
_Note: if an element is added by name as shown above, any matching regex will be ignored_
It is also recommended to ensure multiple patterns don't overlap as order of execution is not guaranteed and can result in some rules being missed.
```go
p.AllowElementsMatching(regex.MustCompile(`^my-element-`))
```
Or add elements as a virtue of adding an attribute:
```go
// Not the recommended pattern, see the recommendation on using .Matching() below
// Note the recommended pattern, see the recommendation on using .Matching() below
p.AllowAttrs("nowrap").OnElements("td", "th")
```
Again, this also supports a regex pattern match alternative:
```go
p.AllowAttrs("nowrap").OnElementsMatching(regex.MustCompile(`^my-element-`))
```
Attributes can either be added to all elements:
```go
p.AllowAttrs("dir").Matching(regexp.MustCompile("(?i)rtl|ltr")).Globally()
@@ -202,6 +218,50 @@ p := bluemonday.UGCPolicy()
p.AllowElements("fieldset", "select", "option")
```
### Inline CSS
Although it's possible to handle inline CSS using `AllowAttrs` with a `Matching` rule, writing a single monolithic regular expression to safely process all inline CSS which you wish to allow is not a trivial task. Instead of attempting to do so, you can allow the `style` attribute on whichever element(s) you desire and use style policies to control and sanitize inline styles.
It is strongly recommended that you use `Matching` (with a suitable regular expression)
`MatchingEnum`, or `MatchingHandler` to ensure each style matches your needs,
but default handlers are supplied for most widely used styles.
Similar to attributes, you can allow specific CSS properties to be set inline:
```go
p.AllowAttrs("style").OnElements("span", "p")
// Allow the 'color' property with valid RGB(A) hex values only (on any element allowed a 'style' attribute)
p.AllowStyles("color").Matching(regexp.MustCompile("(?i)^#([0-9a-f]{3,4}|[0-9a-f]{6}|[0-9a-f]{8})$")).Globally()
```
Additionally, you can allow a CSS property to be set only to an allowed value:
```go
p.AllowAttrs("style").OnElements("span", "p")
// Allow the 'text-decoration' property to be set to 'underline', 'line-through' or 'none'
// on 'span' elements only
p.AllowStyles("text-decoration").MatchingEnum("underline", "line-through", "none").OnElements("span")
```
Or you can specify elements based on a regex pattern match:
```go
p.AllowAttrs("style").OnElementsMatching(regex.MustCompile(`^my-element-`))
// Allow the 'text-decoration' property to be set to 'underline', 'line-through' or 'none'
// on 'span' elements only
p.AllowStyles("text-decoration").MatchingEnum("underline", "line-through", "none").OnElementsMatching(regex.MustCompile(`^my-element-`))
```
If you need more specific checking, you can create a handler that takes in a string and returns a bool to
validate the values for a given property. The string parameter has been
converted to lowercase and unicode code points have been converted.
```go
myHandler := func(value string) bool{
// Validate your input here
return true
}
p.AllowAttrs("style").OnElements("span", "p")
// Allow the 'color' property with values validated by the handler (on any element allowed a 'style' attribute)
p.AllowStyles("color").MatchingHandler(myHandler).Globally()
```
### Links
Links are difficult beasts to sanitise safely and also one of the biggest attack vectors for malicious content.
@@ -220,12 +280,12 @@ We provide some additional global options for safely working with links.
p.RequireParseableURLs(true)
```
If you have enabled parseable URLs then the following option will `AllowRelativeURLs`. By default this is disabled (bluemonday is a whitelist tool... you need to explicitly tell us to permit things) and when disabled it will prevent all local and scheme relative URLs (i.e. `href="localpage.html"`, `href="../home.html"` and even `href="//www.google.com"` are relative):
If you have enabled parseable URLs then the following option will `AllowRelativeURLs`. By default this is disabled (bluemonday is an allowlist tool... you need to explicitly tell us to permit things) and when disabled it will prevent all local and scheme relative URLs (i.e. `href="localpage.html"`, `href="../home.html"` and even `href="//www.google.com"` are relative):
```go
p.AllowRelativeURLs(true)
```
If you have enabled parseable URLs then you can whitelist the schemes (commonly called protocol when thinking of `http` and `https`) that are permitted. Bear in mind that allowing relative URLs in the above option will allow for a blank scheme:
If you have enabled parseable URLs then you can allow the schemes (commonly called protocol when thinking of `http` and `https`) that are permitted. Bear in mind that allowing relative URLs in the above option will allow for a blank scheme:
```go
p.AllowURLSchemes("mailto", "http", "https")
```
@@ -236,7 +296,14 @@ Regardless of whether you have enabled parseable URLs, you can force all URLs to
p.RequireNoFollowOnLinks(true)
```
We provide a convenience method that applies all of the above, but you will still need to whitelist the linkable elements for the URL rules to be applied to:
Similarly, you can force all URLs to have "noreferrer" in their rel attribute.
```go
// This applies to "a" "area" "link" elements that have a "href" attribute
p.RequireNoReferrerOnLinks(true)
```
We provide a convenience method that applies all of the above, but you will still need to allow the linkable elements for the URL rules to be applied to:
```go
p.AllowStandardURLs()
p.AllowAttrs("cite").OnElements("blockquote", "q")
@@ -306,17 +373,18 @@ p.AllowAttrs(
)
```
Both examples exhibit the same issue, they declare attributes but do not then specify whether they are whitelisted globally or only on specific elements (and which elements). Attributes belong to one or more elements, and the policy needs to declare this.
Both examples exhibit the same issue, they declare attributes but do not then specify whether they are allowed globally or only on specific elements (and which elements). Attributes belong to one or more elements, and the policy needs to declare this.
## Limitations
We are not yet including any tools to help whitelist and sanitize CSS. Which means that unless you wish to do the heavy lifting in a single regular expression (inadvisable), **you should not allow the "style" attribute anywhere**.
We are not yet including any tools to help allow and sanitize CSS. Which means that unless you wish to do the heavy lifting in a single regular expression (inadvisable), **you should not allow the "style" attribute anywhere**.
In the same theme, both `<script>` and `<style>` are considered harmful. These elements (and their content) will not be rendered by default, and require you to explicitly set `p.AllowUnsafe(true)`. You should be aware that allowing these elements defeats the purpose of using a HTML sanitizer as you would be explicitly allowing either JavaScript (and any plainly written XSS) and CSS (which can modify a DOM to insert JS), and additionally but limitations in this library mean it is not aware of whether HTML is validly structured and that can allow these elements to bypass some of the safety mechanisms built into the [WhatWG HTML parser standard](https://html.spec.whatwg.org/multipage/parsing.html#parsing-main-inselect).
It is not the job of bluemonday to fix your bad HTML, it is merely the job of bluemonday to prevent malicious HTML getting through. If you have mismatched HTML elements, or non-conforming nesting of elements, those will remain. But if you have well-structured HTML bluemonday will not break it.
## TODO
* Add support for CSS sanitisation to allow some CSS properties based on a whitelist, possibly using the [Gorilla CSS3 scanner](http://www.gorillatoolkit.org/pkg/css/scanner) - PRs welcome so long as testing covers XSS and demonstrates safety first
* Investigate whether devs want to blacklist elements and attributes. This would allow devs to take an existing policy (such as the `bluemonday.UGCPolicy()` ) that encapsulates 90% of what they're looking for but does more than they need, and to remove the extra things they do not want to make it 100% what they want
* Investigate whether devs want a validating HTML mode, in which the HTML elements are not just transformed into a balanced tree (every start tag has a closing tag at the correct depth) but also that elements and character data appear only in their allowed context (i.e. that a `table` element isn't a descendent of a `caption`, that `colgroup`, `thead`, `tbody`, `tfoot` and `tr` are permitted, and that character data is not permitted)

15
vendor/github.com/microcosm-cc/bluemonday/SECURITY.md сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,15 @@
# Security Policy
## Supported Versions
Latest tag and tip are supported.
Older tags remain present but changes result in new tags and are not back ported... please verify any issue against the latest tag and tip.
## Reporting a Vulnerability
Email: <bluemonday@buro9.com>
Bluemonday is pure OSS and not maintained by a company. As such there is no bug bounty program but security issues will be taken seriously and resolved as soon as possible.
The maintainer lives in the United Kingdom and whilst the email is monitored expect a reply or ACK when the maintainer is awake.

2086
vendor/github.com/microcosm-cc/bluemonday/css/handlers.go сгенерированный поставляемый Обычный файл
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

14
vendor/github.com/microcosm-cc/bluemonday/doc.go сгенерированный поставляемый
Просмотреть файл

@@ -28,10 +28,10 @@
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
/*
Package bluemonday provides a way of describing a whitelist of HTML elements
Package bluemonday provides a way of describing an allowlist of HTML elements
and attributes as a policy, and for that policy to be applied to untrusted
strings from users that may contain markup. All elements and attributes not on
the whitelist will be stripped.
the allowlist will be stripped.
The default bluemonday.UGCPolicy().Sanitize() turns this:
@@ -84,21 +84,21 @@ bluemonday is heavily inspired by both the OWASP Java HTML Sanitizer
We ship two default policies, one is bluemonday.StrictPolicy() and can be
thought of as equivalent to stripping all HTML elements and their attributes as
it has nothing on its whitelist.
it has nothing on its allowlist.
The other is bluemonday.UGCPolicy() and allows a broad selection of HTML
elements and attributes that are safe for user generated content. Note that
this policy does not whitelist iframes, object, embed, styles, script, etc.
this policy does not allow iframes, object, embed, styles, script, etc.
The essence of building a policy is to determine which HTML elements and
attributes are considered safe for your scenario. OWASP provide an XSS
prevention cheat sheet ( https://www.google.com/search?q=xss+prevention+cheat+sheet )
to help explain the risks, but essentially:
1. Avoid whitelisting anything other than plain HTML elements
2. Avoid whitelisting `script`, `style`, `iframe`, `object`, `embed`, `base`
1. Avoid allowing anything other than plain HTML elements
2. Avoid allowing `script`, `style`, `iframe`, `object`, `embed`, `base`
elements
3. Avoid whitelisting anything other than plain HTML elements with simple
3. Avoid allowing anything other than plain HTML elements with simple
values that you can match to a regexp
*/
package bluemonday

8
vendor/github.com/microcosm-cc/bluemonday/go.mod сгенерированный поставляемый
Просмотреть файл

@@ -1,5 +1,9 @@
module github.com/microcosm-cc/bluemonday
go 1.9
go 1.16
require golang.org/x/net v0.0.0-20181220203305-927f97764cc3
require (
github.com/aymerick/douceur v0.2.0
github.com/gorilla/css v1.0.0 // indirect
golang.org/x/net v0.0.0-20210614182718-04defd469f4e
)

13
vendor/github.com/microcosm-cc/bluemonday/go.sum сгенерированный поставляемый
Просмотреть файл

@@ -1,2 +1,11 @@
golang.org/x/net v0.0.0-20181220203305-927f97764cc3 h1:eH6Eip3UpmR+yM/qI9Ijluzb1bNv/cAU/n+6l8tRSis=
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY=
github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

10
vendor/github.com/microcosm-cc/bluemonday/helpers.go сгенерированный поставляемый
Просмотреть файл

@@ -135,13 +135,13 @@ func (p *Policy) AllowStandardURLs() {
// Most common URL schemes only
p.AllowURLSchemes("mailto", "http", "https")
// For all anchors we will add rel="nofollow" if it does not already exist
// For linking elements we will add rel="nofollow" if it does not already exist
// This applies to "a" "area" "link"
p.RequireNoFollowOnLinks(true)
}
// AllowStandardAttributes will enable "id", "title" and the language specific
// attributes "dir" and "lang" on all elements that are whitelisted
// attributes "dir" and "lang" on all elements that are allowed
func (p *Policy) AllowStandardAttributes() {
// "dir" "lang" are permitted as both language attributes affect charsets
// and direction of text.
@@ -295,3 +295,9 @@ func (p *Policy) AllowTables() {
CellVerticalAlign,
).OnElements("tbody", "tfoot")
}
func (p *Policy) AllowIFrames(vals ...SandboxValue) {
p.AllowAttrs("sandbox").OnElements("iframe")
p.RequireSandboxOnIFrame(vals...)
}

459
vendor/github.com/microcosm-cc/bluemonday/policy.go сгенерированный поставляемый
Просмотреть файл

@@ -29,13 +29,17 @@
package bluemonday
//TODO sgutzwiller create map of styles to default handlers
//TODO sgutzwiller create handlers for various attributes
import (
"net/url"
"regexp"
"strings"
"github.com/microcosm-cc/bluemonday/css"
)
// Policy encapsulates the whitelist of HTML elements and attributes that will
// Policy encapsulates the allowlist of HTML elements and attributes that will
// be applied to the sanitised HTML.
//
// You should use bluemonday.NewPolicy() to create a blank policy as the
@@ -51,14 +55,28 @@ type Policy struct {
// tag is replaced by a space character.
addSpaces bool
// When true, add rel="nofollow" to HTML anchors
// When true, add rel="nofollow" to HTML a, area, and link tags
requireNoFollow bool
// When true, add rel="nofollow" to HTML anchors
// When true, add rel="nofollow" to HTML a, area, and link tags
// Will add for href="http://foo"
// Will skip for href="/foo" or href="foo"
requireNoFollowFullyQualifiedLinks bool
// When true, add rel="noreferrer" to HTML a, area, and link tags
requireNoReferrer bool
// When true, add rel="noreferrer" to HTML a, area, and link tags
// Will add for href="http://foo"
// Will skip for href="/foo" or href="foo"
requireNoReferrerFullyQualifiedLinks bool
// When true, add crossorigin="anonymous" to HTML audio, img, link, script, and video tags
requireCrossOriginAnonymous bool
// When true, add and filter sandbox attribute on iframe tags
requireSandboxOnIFrame map[string]bool
// When true add target="_blank" to fully qualified links
// Will add for href="http://foo"
// Will skip for href="/foo" or href="foo"
@@ -73,16 +91,31 @@ type Policy struct {
// When true, allow data attributes.
allowDataAttributes bool
// map[htmlElementName]map[htmlAttributeName]attrPolicy
elsAndAttrs map[string]map[string]attrPolicy
// When true, allow comments.
allowComments bool
// map[htmlAttributeName]attrPolicy
globalAttrs map[string]attrPolicy
// map[htmlElementName]map[htmlAttributeName][]attrPolicy
elsAndAttrs map[string]map[string][]attrPolicy
// elsMatchingAndAttrs stores regex based element matches along with attributes
elsMatchingAndAttrs map[*regexp.Regexp]map[string][]attrPolicy
// map[htmlAttributeName][]attrPolicy
globalAttrs map[string][]attrPolicy
// map[htmlElementName]map[cssPropertyName][]stylePolicy
elsAndStyles map[string]map[string][]stylePolicy
// map[regex]map[cssPropertyName][]stylePolicy
elsMatchingAndStyles map[*regexp.Regexp]map[string][]stylePolicy
// map[cssPropertyName][]stylePolicy
globalStyles map[string][]stylePolicy
// If urlPolicy is nil, all URLs with matching schema are allowed.
// Otherwise, only the URLs with matching schema and urlPolicy(url)
// returning true are allowed.
allowURLSchemes map[string]urlPolicy
allowURLSchemes map[string][]urlPolicy
// If an element has had all attributes removed as a result of a policy
// being applied, then the element would be removed from the output.
@@ -93,7 +126,30 @@ type Policy struct {
// be maintained in the output HTML.
setOfElementsAllowedWithoutAttrs map[string]struct{}
// If an element has had all attributes removed as a result of a policy
// being applied, then the element would be removed from the output.
//
// However some elements are valid and have strong layout meaning without
// any attributes, i.e. <table>.
//
// In this case, any element matching a regular expression will be accepted without
// attributes added.
setOfElementsMatchingAllowedWithoutAttrs []*regexp.Regexp
setOfElementsToSkipContent map[string]struct{}
// Permits fundamentally unsafe elements.
//
// If false (default) then elements such as `style` and `script` will not be
// permitted even if declared in a policy. These elements when combined with
// untrusted input cannot be safely handled by bluemonday at this point in
// time.
//
// If true then `style` and `script` would be permitted by bluemonday if a
// policy declares them. However this is not recommended under any circumstance
// and can lead to XSS being rendered thus defeating the purpose of using a
// HTML sanitizer.
allowUnsafe bool
}
type attrPolicy struct {
@@ -103,6 +159,20 @@ type attrPolicy struct {
regexp *regexp.Regexp
}
type stylePolicy struct {
// handler to validate
handler func(string) bool
// optional pattern to match, when not nil the regexp needs to match
// otherwise the property is removed
regexp *regexp.Regexp
// optional list of allowed property values, for properties which
// have a defined list of allowed values; property will be removed
// if the value is not allowed
enum []string
}
type attrPolicyBuilder struct {
p *Policy
@@ -111,23 +181,55 @@ type attrPolicyBuilder struct {
allowEmpty bool
}
type stylePolicyBuilder struct {
p *Policy
propertyNames []string
regexp *regexp.Regexp
enum []string
handler func(string) bool
}
type urlPolicy func(url *url.URL) (allowUrl bool)
type SandboxValue int64
const (
SandboxAllowDownloads SandboxValue = iota
SandboxAllowDownloadsWithoutUserActivation
SandboxAllowForms
SandboxAllowModals
SandboxAllowOrientationLock
SandboxAllowPointerLock
SandboxAllowPopups
SandboxAllowPopupsToEscapeSandbox
SandboxAllowPresentation
SandboxAllowSameOrigin
SandboxAllowScripts
SandboxAllowStorageAccessByUserActivation
SandboxAllowTopNavigation
SandboxAllowTopNavigationByUserActivation
)
// init initializes the maps if this has not been done already
func (p *Policy) init() {
if !p.initialized {
p.elsAndAttrs = make(map[string]map[string]attrPolicy)
p.globalAttrs = make(map[string]attrPolicy)
p.allowURLSchemes = make(map[string]urlPolicy)
p.elsAndAttrs = make(map[string]map[string][]attrPolicy)
p.elsMatchingAndAttrs = make(map[*regexp.Regexp]map[string][]attrPolicy)
p.globalAttrs = make(map[string][]attrPolicy)
p.elsAndStyles = make(map[string]map[string][]stylePolicy)
p.elsMatchingAndStyles = make(map[*regexp.Regexp]map[string][]stylePolicy)
p.globalStyles = make(map[string][]stylePolicy)
p.allowURLSchemes = make(map[string][]urlPolicy)
p.setOfElementsAllowedWithoutAttrs = make(map[string]struct{})
p.setOfElementsToSkipContent = make(map[string]struct{})
p.initialized = true
}
}
// NewPolicy returns a blank policy with nothing whitelisted or permitted. This
// NewPolicy returns a blank policy with nothing allowed or permitted. This
// is the recommended way to start building a policy and you should now use
// AllowAttrs() and/or AllowElements() to construct the whitelist of HTML
// AllowAttrs() and/or AllowElements() to construct the allowlist of HTML
// elements and attributes.
func NewPolicy() *Policy {
@@ -141,7 +243,7 @@ func NewPolicy() *Policy {
// AllowAttrs takes a range of HTML attribute names and returns an
// attribute policy builder that allows you to specify the pattern and scope of
// the whitelisted attribute.
// the allowed attribute.
//
// The attribute policy is only added to the core policy when either Globally()
// or OnElements(...) are called.
@@ -161,7 +263,7 @@ func (p *Policy) AllowAttrs(attrNames ...string) *attrPolicyBuilder {
return &abp
}
// AllowDataAttributes whitelists all data attributes. We can't specify the name
// AllowDataAttributes permits all data attributes. We can't specify the name
// of each attribute exactly as they are customized.
//
// NOTE: These values are not sanitized and applications that evaluate or process
@@ -176,6 +278,22 @@ func (p *Policy) AllowDataAttributes() {
p.allowDataAttributes = true
}
// AllowComments allows comments.
//
// Please note that only one type of comment will be allowed by this, this is the
// the standard HTML comment <!-- --> which includes the use of that to permit
// conditionals as per https://docs.microsoft.com/en-us/previous-versions/windows/internet-explorer/ie-developer/compatibility/ms537512(v=vs.85)?redirectedfrom=MSDN
//
// What is not permitted are CDATA XML comments, as the x/net/html package we depend
// on does not handle this fully and we are not choosing to take on that work:
// https://pkg.go.dev/golang.org/x/net/html#Tokenizer.AllowCDATA . If the x/net/html
// package changes this then these will be considered, otherwise if you AllowComments
// but provide a CDATA comment, then as per the documentation in x/net/html this will
// be treated as a plain HTML comment.
func (p *Policy) AllowComments() {
p.allowComments = true
}
// AllowNoAttrs says that attributes on element are optional.
//
// The attribute policy is only added to the core policy when OnElements(...)
@@ -203,8 +321,7 @@ func (abp *attrPolicyBuilder) AllowNoAttrs() *attrPolicyBuilder {
}
// Matching allows a regular expression to be applied to a nascent attribute
// policy, and returns the attribute policy. Calling this more than once will
// replace the existing regexp.
// policy, and returns the attribute policy.
func (abp *attrPolicyBuilder) Matching(regex *regexp.Regexp) *attrPolicyBuilder {
abp.regexp = regex
@@ -222,7 +339,7 @@ func (abp *attrPolicyBuilder) OnElements(elements ...string) *Policy {
for _, attr := range abp.attrNames {
if _, ok := abp.p.elsAndAttrs[element]; !ok {
abp.p.elsAndAttrs[element] = make(map[string]attrPolicy)
abp.p.elsAndAttrs[element] = make(map[string][]attrPolicy)
}
ap := attrPolicy{}
@@ -230,14 +347,14 @@ func (abp *attrPolicyBuilder) OnElements(elements ...string) *Policy {
ap.regexp = abp.regexp
}
abp.p.elsAndAttrs[element][attr] = ap
abp.p.elsAndAttrs[element][attr] = append(abp.p.elsAndAttrs[element][attr], ap)
}
if abp.allowEmpty {
abp.p.setOfElementsAllowedWithoutAttrs[element] = struct{}{}
if _, ok := abp.p.elsAndAttrs[element]; !ok {
abp.p.elsAndAttrs[element] = make(map[string]attrPolicy)
abp.p.elsAndAttrs[element] = make(map[string][]attrPolicy)
}
}
}
@@ -245,13 +362,37 @@ func (abp *attrPolicyBuilder) OnElements(elements ...string) *Policy {
return abp.p
}
// OnElementsMatching will bind an attribute policy to all elements matching a given regex
// and return the updated policy
func (abp *attrPolicyBuilder) OnElementsMatching(regex *regexp.Regexp) *Policy {
for _, attr := range abp.attrNames {
if _, ok := abp.p.elsMatchingAndAttrs[regex]; !ok {
abp.p.elsMatchingAndAttrs[regex] = make(map[string][]attrPolicy)
}
ap := attrPolicy{}
if abp.regexp != nil {
ap.regexp = abp.regexp
}
abp.p.elsMatchingAndAttrs[regex][attr] = append(abp.p.elsMatchingAndAttrs[regex][attr], ap)
}
if abp.allowEmpty {
abp.p.setOfElementsMatchingAllowedWithoutAttrs = append(abp.p.setOfElementsMatchingAllowedWithoutAttrs, regex)
if _, ok := abp.p.elsMatchingAndAttrs[regex]; !ok {
abp.p.elsMatchingAndAttrs[regex] = make(map[string][]attrPolicy)
}
}
return abp.p
}
// Globally will bind an attribute policy to all HTML elements and return the
// updated policy
func (abp *attrPolicyBuilder) Globally() *Policy {
for _, attr := range abp.attrNames {
if _, ok := abp.p.globalAttrs[attr]; !ok {
abp.p.globalAttrs[attr] = attrPolicy{}
abp.p.globalAttrs[attr] = []attrPolicy{}
}
ap := attrPolicy{}
@@ -259,13 +400,143 @@ func (abp *attrPolicyBuilder) Globally() *Policy {
ap.regexp = abp.regexp
}
abp.p.globalAttrs[attr] = ap
abp.p.globalAttrs[attr] = append(abp.p.globalAttrs[attr], ap)
}
return abp.p
}
// AllowElements will append HTML elements to the whitelist without applying an
// AllowStyles takes a range of CSS property names and returns a
// style policy builder that allows you to specify the pattern and scope of
// the allowed property.
//
// The style policy is only added to the core policy when either Globally()
// or OnElements(...) are called.
func (p *Policy) AllowStyles(propertyNames ...string) *stylePolicyBuilder {
p.init()
abp := stylePolicyBuilder{
p: p,
}
for _, propertyName := range propertyNames {
abp.propertyNames = append(abp.propertyNames, strings.ToLower(propertyName))
}
return &abp
}
// Matching allows a regular expression to be applied to a nascent style
// policy, and returns the style policy.
func (spb *stylePolicyBuilder) Matching(regex *regexp.Regexp) *stylePolicyBuilder {
spb.regexp = regex
return spb
}
// MatchingEnum allows a list of allowed values to be applied to a nascent style
// policy, and returns the style policy.
func (spb *stylePolicyBuilder) MatchingEnum(enum ...string) *stylePolicyBuilder {
spb.enum = enum
return spb
}
// MatchingHandler allows a handler to be applied to a nascent style
// policy, and returns the style policy.
func (spb *stylePolicyBuilder) MatchingHandler(handler func(string) bool) *stylePolicyBuilder {
spb.handler = handler
return spb
}
// OnElements will bind a style policy to a given range of HTML elements
// and return the updated policy
func (spb *stylePolicyBuilder) OnElements(elements ...string) *Policy {
for _, element := range elements {
element = strings.ToLower(element)
for _, attr := range spb.propertyNames {
if _, ok := spb.p.elsAndStyles[element]; !ok {
spb.p.elsAndStyles[element] = make(map[string][]stylePolicy)
}
sp := stylePolicy{}
if spb.handler != nil {
sp.handler = spb.handler
} else if len(spb.enum) > 0 {
sp.enum = spb.enum
} else if spb.regexp != nil {
sp.regexp = spb.regexp
} else {
sp.handler = css.GetDefaultHandler(attr)
}
spb.p.elsAndStyles[element][attr] = append(spb.p.elsAndStyles[element][attr], sp)
}
}
return spb.p
}
// OnElementsMatching will bind a style policy to any HTML elements matching the pattern
// and return the updated policy
func (spb *stylePolicyBuilder) OnElementsMatching(regex *regexp.Regexp) *Policy {
for _, attr := range spb.propertyNames {
if _, ok := spb.p.elsMatchingAndStyles[regex]; !ok {
spb.p.elsMatchingAndStyles[regex] = make(map[string][]stylePolicy)
}
sp := stylePolicy{}
if spb.handler != nil {
sp.handler = spb.handler
} else if len(spb.enum) > 0 {
sp.enum = spb.enum
} else if spb.regexp != nil {
sp.regexp = spb.regexp
} else {
sp.handler = css.GetDefaultHandler(attr)
}
spb.p.elsMatchingAndStyles[regex][attr] = append(spb.p.elsMatchingAndStyles[regex][attr], sp)
}
return spb.p
}
// Globally will bind a style policy to all HTML elements and return the
// updated policy
func (spb *stylePolicyBuilder) Globally() *Policy {
for _, attr := range spb.propertyNames {
if _, ok := spb.p.globalStyles[attr]; !ok {
spb.p.globalStyles[attr] = []stylePolicy{}
}
// Use only one strategy for validating styles, fallback to default
sp := stylePolicy{}
if spb.handler != nil {
sp.handler = spb.handler
} else if len(spb.enum) > 0 {
sp.enum = spb.enum
} else if spb.regexp != nil {
sp.regexp = spb.regexp
} else {
sp.handler = css.GetDefaultHandler(attr)
}
spb.p.globalStyles[attr] = append(spb.p.globalStyles[attr], sp)
}
return spb.p
}
// AllowElements will append HTML elements to the allowlist without applying an
// attribute policy to those elements (the elements are permitted
// sans-attributes)
func (p *Policy) AllowElements(names ...string) *Policy {
@@ -275,15 +546,25 @@ func (p *Policy) AllowElements(names ...string) *Policy {
element = strings.ToLower(element)
if _, ok := p.elsAndAttrs[element]; !ok {
p.elsAndAttrs[element] = make(map[string]attrPolicy)
p.elsAndAttrs[element] = make(map[string][]attrPolicy)
}
}
return p
}
// RequireNoFollowOnLinks will result in all <a> tags having a rel="nofollow"
// added to them if one does not already exist
// AllowElementsMatching will append HTML elements to the allowlist if they
// match a regexp.
func (p *Policy) AllowElementsMatching(regex *regexp.Regexp) *Policy {
p.init()
if _, ok := p.elsMatchingAndAttrs[regex]; !ok {
p.elsMatchingAndAttrs[regex] = make(map[string][]attrPolicy)
}
return p
}
// RequireNoFollowOnLinks will result in all a, area, link tags having a
// rel="nofollow"added to them if one does not already exist
//
// Note: This requires p.RequireParseableURLs(true) and will enable it.
func (p *Policy) RequireNoFollowOnLinks(require bool) *Policy {
@@ -294,9 +575,10 @@ func (p *Policy) RequireNoFollowOnLinks(require bool) *Policy {
return p
}
// RequireNoFollowOnFullyQualifiedLinks will result in all <a> tags that point
// to a non-local destination (i.e. starts with a protocol and has a host)
// having a rel="nofollow" added to them if one does not already exist
// RequireNoFollowOnFullyQualifiedLinks will result in all a, area, and link
// tags that point to a non-local destination (i.e. starts with a protocol and
// has a host) having a rel="nofollow" added to them if one does not already
// exist
//
// Note: This requires p.RequireParseableURLs(true) and will enable it.
func (p *Policy) RequireNoFollowOnFullyQualifiedLinks(require bool) *Policy {
@@ -307,9 +589,45 @@ func (p *Policy) RequireNoFollowOnFullyQualifiedLinks(require bool) *Policy {
return p
}
// AddTargetBlankToFullyQualifiedLinks will result in all <a> tags that point
// to a non-local destination (i.e. starts with a protocol and has a host)
// having a target="_blank" added to them if one does not already exist
// RequireNoReferrerOnLinks will result in all a, area, and link tags having a
// rel="noreferrrer" added to them if one does not already exist
//
// Note: This requires p.RequireParseableURLs(true) and will enable it.
func (p *Policy) RequireNoReferrerOnLinks(require bool) *Policy {
p.requireNoReferrer = require
p.requireParseableURLs = true
return p
}
// RequireNoReferrerOnFullyQualifiedLinks will result in all a, area, and link
// tags that point to a non-local destination (i.e. starts with a protocol and
// has a host) having a rel="noreferrer" added to them if one does not already
// exist
//
// Note: This requires p.RequireParseableURLs(true) and will enable it.
func (p *Policy) RequireNoReferrerOnFullyQualifiedLinks(require bool) *Policy {
p.requireNoReferrerFullyQualifiedLinks = require
p.requireParseableURLs = true
return p
}
// RequireCrossOriginAnonymous will result in all audio, img, link, script, and
// video tags having a crossorigin="anonymous" added to them if one does not
// already exist
func (p *Policy) RequireCrossOriginAnonymous(require bool) *Policy {
p.requireCrossOriginAnonymous = require
return p
}
// AddTargetBlankToFullyQualifiedLinks will result in all a, area and link tags
// that point to a non-local destination (i.e. starts with a protocol and has a
// host) having a target="_blank" added to them if one does not already exist
//
// Note: This requires p.RequireParseableURLs(true) and will enable it.
func (p *Policy) AddTargetBlankToFullyQualifiedLinks(require bool) *Policy {
@@ -347,7 +665,7 @@ func (p *Policy) AllowRelativeURLs(require bool) *Policy {
return p
}
// AllowURLSchemes will append URL schemes to the whitelist
// AllowURLSchemes will append URL schemes to the allowlist
// Example: p.AllowURLSchemes("mailto", "http", "https")
func (p *Policy) AllowURLSchemes(schemes ...string) *Policy {
p.init()
@@ -365,7 +683,7 @@ func (p *Policy) AllowURLSchemes(schemes ...string) *Policy {
}
// AllowURLSchemeWithCustomPolicy will append URL schemes with
// a custom URL policy to the whitelist.
// a custom URL policy to the allowlist.
// Only the URLs with matching schema and urlPolicy(url)
// returning true will be allowed.
func (p *Policy) AllowURLSchemeWithCustomPolicy(
@@ -379,13 +697,65 @@ func (p *Policy) AllowURLSchemeWithCustomPolicy(
scheme = strings.ToLower(scheme)
p.allowURLSchemes[scheme] = urlPolicy
p.allowURLSchemes[scheme] = append(p.allowURLSchemes[scheme], urlPolicy)
return p
}
// RequireSandboxOnIFrame will result in all iframe tags having a sandbox="" tag
// Any sandbox values not specified here will be filtered from the generated HTML
func (p *Policy) RequireSandboxOnIFrame(vals ...SandboxValue) {
p.requireSandboxOnIFrame = make(map[string]bool)
for _, val := range vals {
switch SandboxValue(val) {
case SandboxAllowDownloads:
p.requireSandboxOnIFrame["allow-downloads"] = true
case SandboxAllowDownloadsWithoutUserActivation:
p.requireSandboxOnIFrame["allow-downloads-without-user-activation"] = true
case SandboxAllowForms:
p.requireSandboxOnIFrame["allow-forms"] = true
case SandboxAllowModals:
p.requireSandboxOnIFrame["allow-modals"] = true
case SandboxAllowOrientationLock:
p.requireSandboxOnIFrame["allow-orientation-lock"] = true
case SandboxAllowPointerLock:
p.requireSandboxOnIFrame["allow-pointer-lock"] = true
case SandboxAllowPopups:
p.requireSandboxOnIFrame["allow-popups"] = true
case SandboxAllowPopupsToEscapeSandbox:
p.requireSandboxOnIFrame["allow-popups-to-escape-sandbox"] = true
case SandboxAllowPresentation:
p.requireSandboxOnIFrame["allow-presentation"] = true
case SandboxAllowSameOrigin:
p.requireSandboxOnIFrame["allow-same-origin"] = true
case SandboxAllowScripts:
p.requireSandboxOnIFrame["allow-scripts"] = true
case SandboxAllowStorageAccessByUserActivation:
p.requireSandboxOnIFrame["allow-storage-access-by-user-activation"] = true
case SandboxAllowTopNavigation:
p.requireSandboxOnIFrame["allow-top-navigation"] = true
case SandboxAllowTopNavigationByUserActivation:
p.requireSandboxOnIFrame["allow-top-navigation-by-user-activation"] = true
}
}
}
// AddSpaceWhenStrippingTag states whether to add a single space " " when
// removing tags that are not whitelisted by the policy.
// removing tags that are not allowed by the policy.
//
// This is useful if you expect to strip tags in dense markup and may lose the
// value of whitespace.
@@ -431,6 +801,23 @@ func (p *Policy) AllowElementsContent(names ...string) *Policy {
return p
}
// AllowUnsafe permits fundamentally unsafe elements.
//
// If false (default) then elements such as `style` and `script` will not be
// permitted even if declared in a policy. These elements when combined with
// untrusted input cannot be safely handled by bluemonday at this point in
// time.
//
// If true then `style` and `script` would be permitted by bluemonday if a
// policy declares them. However this is not recommended under any circumstance
// and can lead to XSS being rendered thus defeating the purpose of using a
// HTML sanitizer.
func (p *Policy) AllowUnsafe(allowUnsafe bool) *Policy {
p.init()
p.allowUnsafe = allowUnsafe
return p
}
// addDefaultElementsWithoutAttrs adds the HTML elements that we know are valid
// without any attributes to an internal map.
// i.e. we know that <table> is valid, but <bdo> isn't valid as the "dir" attr

714
vendor/github.com/microcosm-cc/bluemonday/sanitize.go сгенерированный поставляемый
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

11
vendor/github.com/microcosm-cc/bluemonday/stringwriterwriter_go1.12.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,11 @@
//go:build go1.12
// +build go1.12
package bluemonday
import "io"
type stringWriterWriter interface {
io.Writer
io.StringWriter
}

15
vendor/github.com/microcosm-cc/bluemonday/stringwriterwriter_ltgo1.12.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,15 @@
//go:build go1.1 && !go1.12
// +build go1.1,!go1.12
package bluemonday
import "io"
type stringWriterWriter interface {
io.Writer
StringWriter
}
type StringWriter interface {
WriteString(s string) (n int, err error)
}

4
vendor/github.com/miekg/dns/README.md сгенерированный поставляемый
Просмотреть файл

@@ -73,6 +73,10 @@ A not-so-up-to-date-list-that-may-be-actually-current:
* https://github.com/bodgit/tsig
* https://github.com/v2fly/v2ray-core (test only)
* https://kuma.io/
* https://www.misaka.io/services/dns
* https://ping.sx/dig
* https://fleetdeck.io/
* https://github.com/markdingo/autoreverse
Send pull request if you want to be listed here.

2
vendor/github.com/miekg/dns/acceptfunc.go сгенерированный поставляемый
Просмотреть файл

@@ -12,7 +12,7 @@ type MsgAcceptFunc func(dh Header) MsgAcceptAction
//
// * Zero bit isn't zero
//
// * has more than 1 question in the question section
// * does not have exactly 1 question in the question section
//
// * has more than 1 RR in the Answer section
//

127
vendor/github.com/miekg/dns/client.go сгенерированный поставляемый
Просмотреть файл

@@ -18,6 +18,18 @@ const (
tcpIdleTimeout time.Duration = 8 * time.Second
)
func isPacketConn(c net.Conn) bool {
if _, ok := c.(net.PacketConn); !ok {
return false
}
if ua, ok := c.LocalAddr().(*net.UnixAddr); ok {
return ua.Net == "unixgram"
}
return true
}
// A Conn represents a connection to a DNS server.
type Conn struct {
net.Conn // a net.Conn holding the connection
@@ -27,6 +39,14 @@ type Conn struct {
tsigRequestMAC string
}
func (co *Conn) tsigProvider() TsigProvider {
if co.TsigProvider != nil {
return co.TsigProvider
}
// tsigSecretProvider will return ErrSecret if co.TsigSecret is nil.
return tsigSecretProvider(co.TsigSecret)
}
// A Client defines parameters for a DNS client.
type Client struct {
Net string // if "tcp" or "tcp-tls" (DNS over TLS) a TCP query will be initiated, otherwise an UDP one (default is "" for UDP)
@@ -82,6 +102,12 @@ func (c *Client) writeTimeout() time.Duration {
// Dial connects to the address on the named network.
func (c *Client) Dial(address string) (conn *Conn, err error) {
return c.DialContext(context.Background(), address)
}
// DialContext connects to the address on the named network, with a context.Context.
// For TLS over TCP (DoT) the context isn't used yet. This will be enabled when Go 1.18 is released.
func (c *Client) DialContext(ctx context.Context, address string) (conn *Conn, err error) {
// create a new dialer with the appropriate timeout
var d net.Dialer
if c.Dialer == nil {
@@ -101,9 +127,17 @@ func (c *Client) Dial(address string) (conn *Conn, err error) {
if useTLS {
network = strings.TrimSuffix(network, "-tls")
// TODO(miekg): Enable after Go 1.18 is released, to be able to support two prev. releases.
/*
tlsDialer := tls.Dialer{
NetDialer: &d,
Config: c.TLSConfig,
}
conn.Conn, err = tlsDialer.DialContext(ctx, network, address)
*/
conn.Conn, err = tls.DialWithDialer(&d, network, address, c.TLSConfig)
} else {
conn.Conn, err = d.Dial(network, address)
conn.Conn, err = d.DialContext(ctx, network, address)
}
if err != nil {
return nil, err
@@ -139,24 +173,34 @@ func (c *Client) Exchange(m *Msg, address string) (r *Msg, rtt time.Duration, er
// ExchangeWithConn has the same behavior as Exchange, just with a predetermined connection
// that will be used instead of creating a new one.
// Usage pattern with a *dns.Client:
//
// c := new(dns.Client)
// // connection management logic goes here
//
// conn := c.Dial(address)
// in, rtt, err := c.ExchangeWithConn(message, conn)
//
// This allows users of the library to implement their own connection management,
// as opposed to Exchange, which will always use new connections and incur the added overhead
// that entails when using "tcp" and especially "tcp-tls" clients.
// This allows users of the library to implement their own connection management,
// as opposed to Exchange, which will always use new connections and incur the added overhead
// that entails when using "tcp" and especially "tcp-tls" clients.
//
// When the singleflight is set for this client the context is _not_ forwarded to the (shared) exchange, to
// prevent one cancelation from canceling all outstanding requests.
func (c *Client) ExchangeWithConn(m *Msg, conn *Conn) (r *Msg, rtt time.Duration, err error) {
return c.exchangeWithConnContext(context.Background(), m, conn)
}
func (c *Client) exchangeWithConnContext(ctx context.Context, m *Msg, conn *Conn) (r *Msg, rtt time.Duration, err error) {
if !c.SingleInflight {
return c.exchange(m, conn)
return c.exchangeContext(ctx, m, conn)
}
q := m.Question[0]
key := fmt.Sprintf("%s:%d:%d", q.Name, q.Qtype, q.Qclass)
r, rtt, err, shared := c.group.Do(key, func() (*Msg, time.Duration, error) {
return c.exchange(m, conn)
// When we're doing singleflight we don't want one context cancelation, cancel _all_ outstanding queries.
// Hence we ignore the context and use Background().
return c.exchangeContext(context.Background(), m, conn)
})
if r != nil && shared {
r = r.Copy()
@@ -165,8 +209,7 @@ func (c *Client) ExchangeWithConn(m *Msg, conn *Conn) (r *Msg, rtt time.Duration
return r, rtt, err
}
func (c *Client) exchange(m *Msg, co *Conn) (r *Msg, rtt time.Duration, err error) {
func (c *Client) exchangeContext(ctx context.Context, m *Msg, co *Conn) (r *Msg, rtt time.Duration, err error) {
opt := m.IsEdns0()
// If EDNS0 is used use that for size.
if opt != nil && opt.UDPSize() >= MinMsgSize {
@@ -177,16 +220,28 @@ func (c *Client) exchange(m *Msg, co *Conn) (r *Msg, rtt time.Duration, err erro
co.UDPSize = c.UDPSize
}
co.TsigSecret, co.TsigProvider = c.TsigSecret, c.TsigProvider
t := time.Now()
// write with the appropriate write timeout
co.SetWriteDeadline(t.Add(c.getTimeoutForRequest(c.writeTimeout())))
t := time.Now()
writeDeadline := t.Add(c.getTimeoutForRequest(c.writeTimeout()))
readDeadline := t.Add(c.getTimeoutForRequest(c.readTimeout()))
if deadline, ok := ctx.Deadline(); ok {
if deadline.Before(writeDeadline) {
writeDeadline = deadline
}
if deadline.Before(readDeadline) {
readDeadline = deadline
}
}
co.SetWriteDeadline(writeDeadline)
co.SetReadDeadline(readDeadline)
co.TsigSecret, co.TsigProvider = c.TsigSecret, c.TsigProvider
if err = co.WriteMsg(m); err != nil {
return nil, 0, err
}
co.SetReadDeadline(time.Now().Add(c.getTimeoutForRequest(c.readTimeout())))
if _, ok := co.Conn.(net.PacketConn); ok {
if isPacketConn(co.Conn) {
for {
r, err = co.ReadMsg()
// Ignore replies with mismatched IDs because they might be
@@ -224,15 +279,8 @@ func (co *Conn) ReadMsg() (*Msg, error) {
return m, err
}
if t := m.IsTsig(); t != nil {
if co.TsigProvider != nil {
err = tsigVerifyProvider(p, co.TsigProvider, co.tsigRequestMAC, false)
} else {
if _, ok := co.TsigSecret[t.Hdr.Name]; !ok {
return m, ErrSecret
}
// Need to work on the original message p, as that was used to calculate the tsig.
err = TsigVerify(p, co.TsigSecret[t.Hdr.Name], co.tsigRequestMAC, false)
}
// Need to work on the original message p, as that was used to calculate the tsig.
err = tsigVerifyProvider(p, co.tsigProvider(), co.tsigRequestMAC, false)
}
return m, err
}
@@ -247,7 +295,7 @@ func (co *Conn) ReadMsgHeader(hdr *Header) ([]byte, error) {
err error
)
if _, ok := co.Conn.(net.PacketConn); ok {
if isPacketConn(co.Conn) {
if co.UDPSize > MinMsgSize {
p = make([]byte, co.UDPSize)
} else {
@@ -287,7 +335,7 @@ func (co *Conn) Read(p []byte) (n int, err error) {
return 0, ErrConnEmpty
}
if _, ok := co.Conn.(net.PacketConn); ok {
if isPacketConn(co.Conn) {
// UDP connection
return co.Conn.Read(p)
}
@@ -309,17 +357,8 @@ func (co *Conn) Read(p []byte) (n int, err error) {
func (co *Conn) WriteMsg(m *Msg) (err error) {
var out []byte
if t := m.IsTsig(); t != nil {
mac := ""
if co.TsigProvider != nil {
out, mac, err = tsigGenerateProvider(m, co.TsigProvider, co.tsigRequestMAC, false)
} else {
if _, ok := co.TsigSecret[t.Hdr.Name]; !ok {
return ErrSecret
}
out, mac, err = TsigGenerate(m, co.TsigSecret[t.Hdr.Name], co.tsigRequestMAC, false)
}
// Set for the next read, although only used in zone transfers
co.tsigRequestMAC = mac
// Set tsigRequestMAC for the next read, although only used in zone transfers.
out, co.tsigRequestMAC, err = tsigGenerateProvider(m, co.tsigProvider(), co.tsigRequestMAC, false)
} else {
out, err = m.Pack()
}
@@ -336,7 +375,7 @@ func (co *Conn) Write(p []byte) (int, error) {
return 0, &Error{err: "message too large"}
}
if _, ok := co.Conn.(net.PacketConn); ok {
if isPacketConn(co.Conn) {
return co.Conn.Write(p)
}
@@ -435,15 +474,11 @@ func DialTimeoutWithTLS(network, address string, tlsConfig *tls.Config, timeout
// context, if present. If there is both a context deadline and a configured
// timeout on the client, the earliest of the two takes effect.
func (c *Client) ExchangeContext(ctx context.Context, m *Msg, a string) (r *Msg, rtt time.Duration, err error) {
var timeout time.Duration
if deadline, ok := ctx.Deadline(); !ok {
timeout = 0
} else {
timeout = time.Until(deadline)
conn, err := c.DialContext(ctx, a)
if err != nil {
return nil, 0, err
}
// not passing the context to the underlying calls, as the API does not support
// context. For timeouts you should set up Client.Dialer and call Client.Exchange.
// TODO(tmthrgd,miekg): this is a race condition.
c.Dialer = &net.Dialer{Timeout: timeout}
return c.Exchange(m, a)
defer conn.Close()
return c.exchangeWithConnContext(ctx, m, conn)
}

2
vendor/github.com/miekg/dns/doc.go сгенерированный поставляемый
Просмотреть файл

@@ -251,7 +251,7 @@ information.
EDNS0
EDNS0 is an extension mechanism for the DNS defined in RFC 2671 and updated by
RFC 6891. It defines an new RR type, the OPT RR, which is then completely
RFC 6891. It defines a new RR type, the OPT RR, which is then completely
abused.
Basic use pattern for creating an (empty) OPT RR:

Некоторые файлы не были показаны из-за слишком большого количества измененных файлов Показать больше