MM-39315: Bump dependencies (#19039)
Note: We keep splitio/go-client untouched because the dependency is broken. See https://github.com/mattermost/mattermost-server/pull/18604 ```release-note NONE ```
Этот коммит содержится в:
Agniva De Sarker
коммит произвёл
GitHub
GitHub
родитель
fd8fea804b
Коммит
189d447591
15
vendor/github.com/klauspost/compress/s2/.gitignore
сгенерированный
поставляемый
Обычный файл
15
vendor/github.com/klauspost/compress/s2/.gitignore
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,15 @@
|
||||
testdata/bench
|
||||
|
||||
# These explicitly listed benchmark data files are for an obsolete version of
|
||||
# snappy_test.go.
|
||||
testdata/alice29.txt
|
||||
testdata/asyoulik.txt
|
||||
testdata/fireworks.jpeg
|
||||
testdata/geo.protodata
|
||||
testdata/html
|
||||
testdata/html_x_4
|
||||
testdata/kppkn.gtb
|
||||
testdata/lcet10.txt
|
||||
testdata/paper-100k.pdf
|
||||
testdata/plrabn12.txt
|
||||
testdata/urls.10K
|
||||
28
vendor/github.com/klauspost/compress/s2/LICENSE
сгенерированный
поставляемый
Обычный файл
28
vendor/github.com/klauspost/compress/s2/LICENSE
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,28 @@
|
||||
Copyright (c) 2011 The Snappy-Go Authors. All rights reserved.
|
||||
Copyright (c) 2019 Klaus Post. All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
* Neither the name of Google Inc. nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
717
vendor/github.com/klauspost/compress/s2/README.md
сгенерированный
поставляемый
Обычный файл
717
vendor/github.com/klauspost/compress/s2/README.md
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,717 @@
|
||||
# S2 Compression
|
||||
|
||||
S2 is an extension of [Snappy](https://github.com/google/snappy).
|
||||
|
||||
S2 is aimed for high throughput, which is why it features concurrent compression for bigger payloads.
|
||||
|
||||
Decoding is compatible with Snappy compressed content, but content compressed with S2 cannot be decompressed by Snappy.
|
||||
This means that S2 can seamlessly replace Snappy without converting compressed content.
|
||||
|
||||
S2 can produce Snappy compatible output, faster and better than Snappy.
|
||||
If you want full benefit of the changes you should use s2 without Snappy compatibility.
|
||||
|
||||
S2 is designed to have high throughput on content that cannot be compressed.
|
||||
This is important, so you don't have to worry about spending CPU cycles on already compressed data.
|
||||
|
||||
## Benefits over Snappy
|
||||
|
||||
* Better compression
|
||||
* Adjustable compression (3 levels)
|
||||
* Concurrent stream compression
|
||||
* Faster decompression, even for Snappy compatible content
|
||||
* Ability to quickly skip forward in compressed stream
|
||||
* Compatible with reading Snappy compressed content
|
||||
* Smaller block size overhead on incompressible blocks
|
||||
* Block concatenation
|
||||
* Uncompressed stream mode
|
||||
* Automatic stream size padding
|
||||
* Snappy compatible block compression
|
||||
|
||||
## Drawbacks over Snappy
|
||||
|
||||
* Not optimized for 32 bit systems.
|
||||
* Streams use slightly more memory due to larger blocks and concurrency (configurable).
|
||||
|
||||
# Usage
|
||||
|
||||
Installation: `go get -u github.com/klauspost/compress/s2`
|
||||
|
||||
Full package documentation:
|
||||
|
||||
[![godoc][1]][2]
|
||||
|
||||
[1]: https://godoc.org/github.com/klauspost/compress?status.svg
|
||||
[2]: https://godoc.org/github.com/klauspost/compress/s2
|
||||
|
||||
## Compression
|
||||
|
||||
```Go
|
||||
func EncodeStream(src io.Reader, dst io.Writer) error {
|
||||
enc := s2.NewWriter(dst)
|
||||
_, err := io.Copy(enc, src)
|
||||
if err != nil {
|
||||
enc.Close()
|
||||
return err
|
||||
}
|
||||
// Blocks until compression is done.
|
||||
return enc.Close()
|
||||
}
|
||||
```
|
||||
|
||||
You should always call `enc.Close()`, otherwise you will leak resources and your encode will be incomplete.
|
||||
|
||||
For the best throughput, you should attempt to reuse the `Writer` using the `Reset()` method.
|
||||
|
||||
The Writer in S2 is always buffered, therefore `NewBufferedWriter` in Snappy can be replaced with `NewWriter` in S2.
|
||||
It is possible to flush any buffered data using the `Flush()` method.
|
||||
This will block until all data sent to the encoder has been written to the output.
|
||||
|
||||
S2 also supports the `io.ReaderFrom` interface, which will consume all input from a reader.
|
||||
|
||||
As a final method to compress data, if you have a single block of data you would like to have encoded as a stream,
|
||||
a slightly more efficient method is to use the `EncodeBuffer` method.
|
||||
This will take ownership of the buffer until the stream is closed.
|
||||
|
||||
```Go
|
||||
func EncodeStream(src []byte, dst io.Writer) error {
|
||||
enc := s2.NewWriter(dst)
|
||||
// The encoder owns the buffer until Flush or Close is called.
|
||||
err := enc.EncodeBuffer(buf)
|
||||
if err != nil {
|
||||
enc.Close()
|
||||
return err
|
||||
}
|
||||
// Blocks until compression is done.
|
||||
return enc.Close()
|
||||
}
|
||||
```
|
||||
|
||||
Each call to `EncodeBuffer` will result in discrete blocks being created without buffering,
|
||||
so it should only be used a single time per stream.
|
||||
If you need to write several blocks, you should use the regular io.Writer interface.
|
||||
|
||||
|
||||
## Decompression
|
||||
|
||||
```Go
|
||||
func DecodeStream(src io.Reader, dst io.Writer) error {
|
||||
dec := s2.NewReader(src)
|
||||
_, err := io.Copy(dst, dec)
|
||||
return err
|
||||
}
|
||||
```
|
||||
|
||||
Similar to the Writer, a Reader can be reused using the `Reset` method.
|
||||
|
||||
For the best possible throughput, there is a `EncodeBuffer(buf []byte)` function available.
|
||||
However, it requires that the provided buffer isn't used after it is handed over to S2 and until the stream is flushed or closed.
|
||||
|
||||
For smaller data blocks, there is also a non-streaming interface: `Encode()`, `EncodeBetter()` and `Decode()`.
|
||||
Do however note that these functions (similar to Snappy) does not provide validation of data,
|
||||
so data corruption may be undetected. Stream encoding provides CRC checks of data.
|
||||
|
||||
It is possible to efficiently skip forward in a compressed stream using the `Skip()` method.
|
||||
For big skips the decompressor is able to skip blocks without decompressing them.
|
||||
|
||||
## Single Blocks
|
||||
|
||||
Similar to Snappy S2 offers single block compression.
|
||||
Blocks do not offer the same flexibility and safety as streams,
|
||||
but may be preferable for very small payloads, less than 100K.
|
||||
|
||||
Using a simple `dst := s2.Encode(nil, src)` will compress `src` and return the compressed result.
|
||||
It is possible to provide a destination buffer.
|
||||
If the buffer has a capacity of `s2.MaxEncodedLen(len(src))` it will be used.
|
||||
If not a new will be allocated.
|
||||
|
||||
Alternatively `EncodeBetter`/`EncodeBest` can also be used for better, but slightly slower compression.
|
||||
|
||||
Similarly to decompress a block you can use `dst, err := s2.Decode(nil, src)`.
|
||||
Again an optional destination buffer can be supplied.
|
||||
The `s2.DecodedLen(src)` can be used to get the minimum capacity needed.
|
||||
If that is not satisfied a new buffer will be allocated.
|
||||
|
||||
Block function always operate on a single goroutine since it should only be used for small payloads.
|
||||
|
||||
# Commandline tools
|
||||
|
||||
Some very simply commandline tools are provided; `s2c` for compression and `s2d` for decompression.
|
||||
|
||||
Binaries can be downloaded on the [Releases Page](https://github.com/klauspost/compress/releases).
|
||||
|
||||
Installing then requires Go to be installed. To install them, use:
|
||||
|
||||
`go install github.com/klauspost/compress/s2/cmd/s2c && go install github.com/klauspost/compress/s2/cmd/s2d`
|
||||
|
||||
To build binaries to the current folder use:
|
||||
|
||||
`go build github.com/klauspost/compress/s2/cmd/s2c && go build github.com/klauspost/compress/s2/cmd/s2d`
|
||||
|
||||
|
||||
## s2c
|
||||
|
||||
```
|
||||
Usage: s2c [options] file1 file2
|
||||
|
||||
Compresses all files supplied as input separately.
|
||||
Output files are written as 'filename.ext.s2' or 'filename.ext.snappy'.
|
||||
By default output files will be overwritten.
|
||||
Use - as the only file name to read from stdin and write to stdout.
|
||||
|
||||
Wildcards are accepted: testdir/*.txt will compress all files in testdir ending with .txt
|
||||
Directories can be wildcards as well. testdir/*/*.txt will match testdir/subdir/b.txt
|
||||
|
||||
File names beginning with 'http://' and 'https://' will be downloaded and compressed.
|
||||
Only http response code 200 is accepted.
|
||||
|
||||
Options:
|
||||
-bench int
|
||||
Run benchmark n times. No output will be written
|
||||
-blocksize string
|
||||
Max block size. Examples: 64K, 256K, 1M, 4M. Must be power of two and <= 4MB (default "4M")
|
||||
-c Write all output to stdout. Multiple input files will be concatenated
|
||||
-cpu int
|
||||
Compress using this amount of threads (default 32)
|
||||
-faster
|
||||
Compress faster, but with a minor compression loss
|
||||
-help
|
||||
Display help
|
||||
-o string
|
||||
Write output to another file. Single input file only
|
||||
-pad string
|
||||
Pad size to a multiple of this value, Examples: 500, 64K, 256K, 1M, 4M, etc (default "1")
|
||||
-q Don't write any output to terminal, except errors
|
||||
-rm
|
||||
Delete source file(s) after successful compression
|
||||
-safe
|
||||
Do not overwrite output files
|
||||
-slower
|
||||
Compress more, but a lot slower
|
||||
-snappy
|
||||
Generate Snappy compatible output stream
|
||||
-verify
|
||||
Verify written files
|
||||
|
||||
```
|
||||
|
||||
## s2d
|
||||
|
||||
```
|
||||
Usage: s2d [options] file1 file2
|
||||
|
||||
Decompresses all files supplied as input. Input files must end with '.s2' or '.snappy'.
|
||||
Output file names have the extension removed. By default output files will be overwritten.
|
||||
Use - as the only file name to read from stdin and write to stdout.
|
||||
|
||||
Wildcards are accepted: testdir/*.txt will compress all files in testdir ending with .txt
|
||||
Directories can be wildcards as well. testdir/*/*.txt will match testdir/subdir/b.txt
|
||||
|
||||
File names beginning with 'http://' and 'https://' will be downloaded and decompressed.
|
||||
Extensions on downloaded files are ignored. Only http response code 200 is accepted.
|
||||
|
||||
Options:
|
||||
-bench int
|
||||
Run benchmark n times. No output will be written
|
||||
-c Write all output to stdout. Multiple input files will be concatenated
|
||||
-help
|
||||
Display help
|
||||
-o string
|
||||
Write output to another file. Single input file only
|
||||
-q Don't write any output to terminal, except errors
|
||||
-rm
|
||||
Delete source file(s) after successful decompression
|
||||
-safe
|
||||
Do not overwrite output files
|
||||
-verify
|
||||
Verify files, but do not write output
|
||||
```
|
||||
|
||||
## s2sx: self-extracting archives
|
||||
|
||||
s2sx allows creating self-extracting archives with no dependencies.
|
||||
|
||||
By default, executables are created for the same platforms as the host os,
|
||||
but this can be overridden with `-os` and `-arch` parameters.
|
||||
|
||||
Extracted files have 0666 permissions, except when untar option used.
|
||||
|
||||
```
|
||||
Usage: s2sx [options] file1 file2
|
||||
|
||||
Compresses all files supplied as input separately.
|
||||
If files have '.s2' extension they are assumed to be compressed already.
|
||||
Output files are written as 'filename.s2sx' and with '.exe' for windows targets.
|
||||
If output is big, an additional file with ".more" is written. This must be included as well.
|
||||
By default output files will be overwritten.
|
||||
|
||||
Wildcards are accepted: testdir/*.txt will compress all files in testdir ending with .txt
|
||||
Directories can be wildcards as well. testdir/*/*.txt will match testdir/subdir/b.txt
|
||||
|
||||
Options:
|
||||
-arch string
|
||||
Destination architecture (default "amd64")
|
||||
-c Write all output to stdout. Multiple input files will be concatenated
|
||||
-cpu int
|
||||
Compress using this amount of threads (default 32)
|
||||
-help
|
||||
Display help
|
||||
-max string
|
||||
Maximum executable size. Rest will be written to another file. (default "1G")
|
||||
-os string
|
||||
Destination operating system (default "windows")
|
||||
-q Don't write any output to terminal, except errors
|
||||
-rm
|
||||
Delete source file(s) after successful compression
|
||||
-safe
|
||||
Do not overwrite output files
|
||||
-untar
|
||||
Untar on destination
|
||||
```
|
||||
|
||||
Available platforms are:
|
||||
|
||||
* darwin-amd64
|
||||
* darwin-arm64
|
||||
* linux-amd64
|
||||
* linux-arm
|
||||
* linux-arm64
|
||||
* linux-mips64
|
||||
* linux-ppc64le
|
||||
* windows-386
|
||||
* windows-amd64
|
||||
|
||||
By default, there is a size limit of 1GB for the output executable.
|
||||
|
||||
When this is exceeded the remaining file content is written to a file called
|
||||
output+`.more`. This file must be included for a successful extraction and
|
||||
placed alongside the executable for a successful extraction.
|
||||
|
||||
This file *must* have the same name as the executable, so if the executable is renamed,
|
||||
so must the `.more` file.
|
||||
|
||||
This functionality is disabled with stdin/stdout.
|
||||
|
||||
### Self-extracting TAR files
|
||||
|
||||
If you wrap a TAR file you can specify `-untar` to make it untar on the destination host.
|
||||
|
||||
Files are extracted to the current folder with the path specified in the tar file.
|
||||
|
||||
Note that tar files are not validated before they are wrapped.
|
||||
|
||||
For security reasons files that move below the root folder are not allowed.
|
||||
|
||||
# Performance
|
||||
|
||||
This section will focus on comparisons to Snappy.
|
||||
This package is solely aimed at replacing Snappy as a high speed compression package.
|
||||
If you are mainly looking for better compression [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd)
|
||||
gives better compression, but typically at speeds slightly below "better" mode in this package.
|
||||
|
||||
Compression is increased compared to Snappy, mostly around 5-20% and the throughput is typically 25-40% increased (single threaded) compared to the Snappy Go implementation.
|
||||
|
||||
Streams are concurrently compressed. The stream will be distributed among all available CPU cores for the best possible throughput.
|
||||
|
||||
A "better" compression mode is also available. This allows to trade a bit of speed for a minor compression gain.
|
||||
The content compressed in this mode is fully compatible with the standard decoder.
|
||||
|
||||
Snappy vs S2 **compression** speed on 16 core (32 thread) computer, using all threads and a single thread (1 CPU):
|
||||
|
||||
| File | S2 speed | S2 Throughput | S2 % smaller | S2 "better" | "better" throughput | "better" % smaller |
|
||||
|-----------------------------------------------------------------------------------------------------|----------|---------------|--------------|-------------|---------------------|--------------------|
|
||||
| [rawstudio-mint14.tar](https://files.klauspost.com/compress/rawstudio-mint14.7z) | 12.70x | 10556 MB/s | 7.35% | 4.15x | 3455 MB/s | 12.79% |
|
||||
| (1 CPU) | 1.14x | 948 MB/s | - | 0.42x | 349 MB/s | - |
|
||||
| [github-june-2days-2019.json](https://files.klauspost.com/compress/github-june-2days-2019.json.zst) | 17.13x | 14484 MB/s | 31.60% | 10.09x | 8533 MB/s | 37.71% |
|
||||
| (1 CPU) | 1.33x | 1127 MB/s | - | 0.70x | 589 MB/s | - |
|
||||
| [github-ranks-backup.bin](https://files.klauspost.com/compress/github-ranks-backup.bin.zst) | 15.14x | 12000 MB/s | -5.79% | 6.59x | 5223 MB/s | 5.80% |
|
||||
| (1 CPU) | 1.11x | 877 MB/s | - | 0.47x | 370 MB/s | - |
|
||||
| [consensus.db.10gb](https://files.klauspost.com/compress/consensus.db.10gb.zst) | 14.62x | 12116 MB/s | 15.90% | 5.35x | 4430 MB/s | 16.08% |
|
||||
| (1 CPU) | 1.38x | 1146 MB/s | - | 0.38x | 312 MB/s | - |
|
||||
| [adresser.json](https://files.klauspost.com/compress/adresser.json.zst) | 8.83x | 17579 MB/s | 43.86% | 6.54x | 13011 MB/s | 47.23% |
|
||||
| (1 CPU) | 1.14x | 2259 MB/s | - | 0.74x | 1475 MB/s | - |
|
||||
| [gob-stream](https://files.klauspost.com/compress/gob-stream.7z) | 16.72x | 14019 MB/s | 24.02% | 10.11x | 8477 MB/s | 30.48% |
|
||||
| (1 CPU) | 1.24x | 1043 MB/s | - | 0.70x | 586 MB/s | - |
|
||||
| [10gb.tar](http://mattmahoney.net/dc/10gb.html) | 13.33x | 9254 MB/s | 1.84% | 6.75x | 4686 MB/s | 6.72% |
|
||||
| (1 CPU) | 0.97x | 672 MB/s | - | 0.53x | 366 MB/s | - |
|
||||
| sharnd.out.2gb | 2.11x | 12639 MB/s | 0.01% | 1.98x | 11833 MB/s | 0.01% |
|
||||
| (1 CPU) | 0.93x | 5594 MB/s | - | 1.34x | 8030 MB/s | - |
|
||||
| [enwik9](http://mattmahoney.net/dc/textdata.html) | 19.34x | 8220 MB/s | 3.98% | 7.87x | 3345 MB/s | 15.82% |
|
||||
| (1 CPU) | 1.06x | 452 MB/s | - | 0.50x | 213 MB/s | - |
|
||||
| [silesia.tar](http://sun.aei.polsl.pl/~sdeor/corpus/silesia.zip) | 10.48x | 6124 MB/s | 5.67% | 3.76x | 2197 MB/s | 12.60% |
|
||||
| (1 CPU) | 0.97x | 568 MB/s | - | 0.46x | 271 MB/s | - |
|
||||
| [enwik10](https://encode.su/threads/3315-enwik10-benchmark-results) | 21.07x | 9020 MB/s | 6.36% | 6.91x | 2959 MB/s | 16.95% |
|
||||
| (1 CPU) | 1.07x | 460 MB/s | - | 0.51x | 220 MB/s | - |
|
||||
|
||||
### Legend
|
||||
|
||||
* `S2 speed`: Speed of S2 compared to Snappy, using 16 cores and 1 core.
|
||||
* `S2 throughput`: Throughput of S2 in MB/s.
|
||||
* `S2 % smaller`: How many percent of the Snappy output size is S2 better.
|
||||
* `S2 "better"`: Speed when enabling "better" compression mode in S2 compared to Snappy.
|
||||
* `"better" throughput`: Speed when enabling "better" compression mode in S2 compared to Snappy.
|
||||
* `"better" % smaller`: How many percent of the Snappy output size is S2 better when using "better" compression.
|
||||
|
||||
There is a good speedup across the board when using a single thread and a significant speedup when using multiple threads.
|
||||
|
||||
Machine generated data gets by far the biggest compression boost, with size being being reduced by up to 45% of Snappy size.
|
||||
|
||||
The "better" compression mode sees a good improvement in all cases, but usually at a performance cost.
|
||||
|
||||
Incompressible content (`sharnd.out.2gb`, 2GB random data) sees the smallest speedup.
|
||||
This is likely dominated by synchronization overhead, which is confirmed by the fact that single threaded performance is higher (see above).
|
||||
|
||||
## Decompression
|
||||
|
||||
S2 attempts to create content that is also fast to decompress, except in "better" mode where the smallest representation is used.
|
||||
|
||||
S2 vs Snappy **decompression** speed. Both operating on single core:
|
||||
|
||||
| File | S2 Throughput | vs. Snappy | Better Throughput | vs. Snappy |
|
||||
|-----------------------------------------------------------------------------------------------------|---------------|------------|-------------------|------------|
|
||||
| [rawstudio-mint14.tar](https://files.klauspost.com/compress/rawstudio-mint14.7z) | 2117 MB/s | 1.14x | 1738 MB/s | 0.94x |
|
||||
| [github-june-2days-2019.json](https://files.klauspost.com/compress/github-june-2days-2019.json.zst) | 2401 MB/s | 1.25x | 2307 MB/s | 1.20x |
|
||||
| [github-ranks-backup.bin](https://files.klauspost.com/compress/github-ranks-backup.bin.zst) | 2075 MB/s | 0.98x | 1764 MB/s | 0.83x |
|
||||
| [consensus.db.10gb](https://files.klauspost.com/compress/consensus.db.10gb.zst) | 2967 MB/s | 1.05x | 2885 MB/s | 1.02x |
|
||||
| [adresser.json](https://files.klauspost.com/compress/adresser.json.zst) | 4141 MB/s | 1.07x | 4184 MB/s | 1.08x |
|
||||
| [gob-stream](https://files.klauspost.com/compress/gob-stream.7z) | 2264 MB/s | 1.12x | 2185 MB/s | 1.08x |
|
||||
| [10gb.tar](http://mattmahoney.net/dc/10gb.html) | 1525 MB/s | 1.03x | 1347 MB/s | 0.91x |
|
||||
| sharnd.out.2gb | 3813 MB/s | 0.79x | 3900 MB/s | 0.81x |
|
||||
| [enwik9](http://mattmahoney.net/dc/textdata.html) | 1246 MB/s | 1.29x | 967 MB/s | 1.00x |
|
||||
| [silesia.tar](http://sun.aei.polsl.pl/~sdeor/corpus/silesia.zip) | 1433 MB/s | 1.12x | 1203 MB/s | 0.94x |
|
||||
| [enwik10](https://encode.su/threads/3315-enwik10-benchmark-results) | 1284 MB/s | 1.32x | 1010 MB/s | 1.04x |
|
||||
|
||||
### Legend
|
||||
|
||||
* `S2 Throughput`: Decompression speed of S2 encoded content.
|
||||
* `Better Throughput`: Decompression speed of S2 "better" encoded content.
|
||||
* `vs Snappy`: Decompression speed of S2 "better" mode compared to Snappy and absolute speed.
|
||||
|
||||
|
||||
While the decompression code hasn't changed, there is a significant speedup in decompression speed.
|
||||
S2 prefers longer matches and will typically only find matches that are 6 bytes or longer.
|
||||
While this reduces compression a bit, it improves decompression speed.
|
||||
|
||||
The "better" compression mode will actively look for shorter matches, which is why it has a decompression speed quite similar to Snappy.
|
||||
|
||||
Without assembly decompression is also very fast; single goroutine decompression speed. No assembly:
|
||||
|
||||
| File | S2 Throughput | S2 throughput |
|
||||
|--------------------------------|--------------|---------------|
|
||||
| consensus.db.10gb.s2 | 1.84x | 2289.8 MB/s |
|
||||
| 10gb.tar.s2 | 1.30x | 867.07 MB/s |
|
||||
| rawstudio-mint14.tar.s2 | 1.66x | 1329.65 MB/s |
|
||||
| github-june-2days-2019.json.s2 | 2.36x | 1831.59 MB/s |
|
||||
| github-ranks-backup.bin.s2 | 1.73x | 1390.7 MB/s |
|
||||
| enwik9.s2 | 1.67x | 681.53 MB/s |
|
||||
| adresser.json.s2 | 3.41x | 4230.53 MB/s |
|
||||
| silesia.tar.s2 | 1.52x | 811.58 |
|
||||
|
||||
Even though S2 typically compresses better than Snappy, decompression speed is always better.
|
||||
|
||||
## Block compression
|
||||
|
||||
|
||||
When compressing blocks no concurrent compression is performed just as Snappy.
|
||||
This is because blocks are for smaller payloads and generally will not benefit from concurrent compression.
|
||||
|
||||
An important change is that incompressible blocks will not be more than at most 10 bytes bigger than the input.
|
||||
In rare, worst case scenario Snappy blocks could be significantly bigger than the input.
|
||||
|
||||
### Mixed content blocks
|
||||
|
||||
The most reliable is a wide dataset.
|
||||
For this we use [`webdevdata.org-2015-01-07-subset`](https://files.klauspost.com/compress/webdevdata.org-2015-01-07-4GB-subset.7z),
|
||||
53927 files, total input size: 4,014,735,833 bytes. Single goroutine used.
|
||||
|
||||
| * | Input | Output | Reduction | MB/s |
|
||||
|-------------------|------------|------------|-----------|--------|
|
||||
| S2 | 4014735833 | 1059723369 | 73.60% | **934.34** |
|
||||
| S2 Better | 4014735833 | 969670507 | 75.85% | 532.70 |
|
||||
| S2 Best | 4014735833 | 906625668 | **77.85%** | 46.84 |
|
||||
| Snappy | 4014735833 | 1128706759 | 71.89% | 762.59 |
|
||||
| S2, Snappy Output | 4014735833 | 1093821420 | 72.75% | 908.60 |
|
||||
| LZ4 | 4014735833 | 1079259294 | 73.12% | 526.94 |
|
||||
|
||||
S2 delivers both the best single threaded throughput with regular mode and the best compression rate with "best".
|
||||
"Better" mode provides the same compression speed as LZ4 with better compression ratio.
|
||||
|
||||
When outputting Snappy compatible output it still delivers better throughput (150MB/s more) and better compression.
|
||||
|
||||
As can be seen from the other benchmarks decompression should also be easier on the S2 generated output.
|
||||
|
||||
Though they cannot be compared due to different decompression speeds here are the speed/size comparisons for
|
||||
other Go compressors:
|
||||
|
||||
| * | Input | Output | Reduction | MB/s |
|
||||
|-------------------|------------|------------|-----------|--------|
|
||||
| Zstd Fastest (Go) | 4014735833 | 794608518 | 80.21% | 236.04 |
|
||||
| Zstd Best (Go) | 4014735833 | 704603356 | 82.45% | 35.63 |
|
||||
| Deflate (Go) l1 | 4014735833 | 871294239 | 78.30% | 214.04 |
|
||||
| Deflate (Go) l9 | 4014735833 | 730389060 | 81.81% | 41.17 |
|
||||
|
||||
### Standard block compression
|
||||
|
||||
Benchmarking single block performance is subject to a lot more variation since it only tests a limited number of file patterns.
|
||||
So individual benchmarks should only be seen as a guideline and the overall picture is more important.
|
||||
|
||||
These micro-benchmarks are with data in cache and trained branch predictors. For a more realistic benchmark see the mixed content above.
|
||||
|
||||
Block compression. Parallel benchmark running on 16 cores, 16 goroutines.
|
||||
|
||||
AMD64 assembly is use for both S2 and Snappy.
|
||||
|
||||
| Absolute Perf | Snappy size | S2 Size | Snappy Speed | S2 Speed | Snappy dec | S2 dec |
|
||||
|-----------------------|-------------|---------|--------------|-------------|-------------|-------------|
|
||||
| html | 22843 | 21111 | 16246 MB/s | 17438 MB/s | 40972 MB/s | 49263 MB/s |
|
||||
| urls.10K | 335492 | 287326 | 7943 MB/s | 9693 MB/s | 22523 MB/s | 26484 MB/s |
|
||||
| fireworks.jpeg | 123034 | 123100 | 349544 MB/s | 273889 MB/s | 718321 MB/s | 827552 MB/s |
|
||||
| fireworks.jpeg (200B) | 146 | 155 | 8869 MB/s | 17773 MB/s | 33691 MB/s | 52421 MB/s |
|
||||
| paper-100k.pdf | 85304 | 84459 | 167546 MB/s | 101263 MB/s | 326905 MB/s | 291944 MB/s |
|
||||
| html_x_4 | 92234 | 21113 | 15194 MB/s | 50670 MB/s | 30843 MB/s | 32217 MB/s |
|
||||
| alice29.txt | 88034 | 85975 | 5936 MB/s | 6139 MB/s | 12882 MB/s | 20044 MB/s |
|
||||
| asyoulik.txt | 77503 | 79650 | 5517 MB/s | 6366 MB/s | 12735 MB/s | 22806 MB/s |
|
||||
| lcet10.txt | 234661 | 220670 | 6235 MB/s | 6067 MB/s | 14519 MB/s | 18697 MB/s |
|
||||
| plrabn12.txt | 319267 | 317985 | 5159 MB/s | 5726 MB/s | 11923 MB/s | 19901 MB/s |
|
||||
| geo.protodata | 23335 | 18690 | 21220 MB/s | 26529 MB/s | 56271 MB/s | 62540 MB/s |
|
||||
| kppkn.gtb | 69526 | 65312 | 9732 MB/s | 8559 MB/s | 18491 MB/s | 18969 MB/s |
|
||||
| alice29.txt (128B) | 80 | 82 | 6691 MB/s | 15489 MB/s | 31883 MB/s | 38874 MB/s |
|
||||
| alice29.txt (1000B) | 774 | 774 | 12204 MB/s | 13000 MB/s | 48056 MB/s | 52341 MB/s |
|
||||
| alice29.txt (10000B) | 6648 | 6933 | 10044 MB/s | 12806 MB/s | 32378 MB/s | 46322 MB/s |
|
||||
| alice29.txt (20000B) | 12686 | 13574 | 7733 MB/s | 11210 MB/s | 30566 MB/s | 58969 MB/s |
|
||||
|
||||
|
||||
| Relative Perf | Snappy size | S2 size improved | S2 Speed | S2 Dec Speed |
|
||||
|-----------------------|-------------|------------------|----------|--------------|
|
||||
| html | 22.31% | 7.58% | 1.07x | 1.20x |
|
||||
| urls.10K | 47.78% | 14.36% | 1.22x | 1.18x |
|
||||
| fireworks.jpeg | 99.95% | -0.05% | 0.78x | 1.15x |
|
||||
| fireworks.jpeg (200B) | 73.00% | -6.16% | 2.00x | 1.56x |
|
||||
| paper-100k.pdf | 83.30% | 0.99% | 0.60x | 0.89x |
|
||||
| html_x_4 | 22.52% | 77.11% | 3.33x | 1.04x |
|
||||
| alice29.txt | 57.88% | 2.34% | 1.03x | 1.56x |
|
||||
| asyoulik.txt | 61.91% | -2.77% | 1.15x | 1.79x |
|
||||
| lcet10.txt | 54.99% | 5.96% | 0.97x | 1.29x |
|
||||
| plrabn12.txt | 66.26% | 0.40% | 1.11x | 1.67x |
|
||||
| geo.protodata | 19.68% | 19.91% | 1.25x | 1.11x |
|
||||
| kppkn.gtb | 37.72% | 6.06% | 0.88x | 1.03x |
|
||||
| alice29.txt (128B) | 62.50% | -2.50% | 2.31x | 1.22x |
|
||||
| alice29.txt (1000B) | 77.40% | 0.00% | 1.07x | 1.09x |
|
||||
| alice29.txt (10000B) | 66.48% | -4.29% | 1.27x | 1.43x |
|
||||
| alice29.txt (20000B) | 63.43% | -7.00% | 1.45x | 1.93x |
|
||||
|
||||
Speed is generally at or above Snappy. Small blocks gets a significant speedup, although at the expense of size.
|
||||
|
||||
Decompression speed is better than Snappy, except in one case.
|
||||
|
||||
Since payloads are very small the variance in terms of size is rather big, so they should only be seen as a general guideline.
|
||||
|
||||
Size is on average around Snappy, but varies on content type.
|
||||
In cases where compression is worse, it usually is compensated by a speed boost.
|
||||
|
||||
|
||||
### Better compression
|
||||
|
||||
Benchmarking single block performance is subject to a lot more variation since it only tests a limited number of file patterns.
|
||||
So individual benchmarks should only be seen as a guideline and the overall picture is more important.
|
||||
|
||||
| Absolute Perf | Snappy size | Better Size | Snappy Speed | Better Speed | Snappy dec | Better dec |
|
||||
|-----------------------|-------------|-------------|--------------|--------------|-------------|-------------|
|
||||
| html | 22843 | 19833 | 16246 MB/s | 7731 MB/s | 40972 MB/s | 40292 MB/s |
|
||||
| urls.10K | 335492 | 253529 | 7943 MB/s | 3980 MB/s | 22523 MB/s | 20981 MB/s |
|
||||
| fireworks.jpeg | 123034 | 123100 | 349544 MB/s | 9760 MB/s | 718321 MB/s | 823698 MB/s |
|
||||
| fireworks.jpeg (200B) | 146 | 142 | 8869 MB/s | 594 MB/s | 33691 MB/s | 30101 MB/s |
|
||||
| paper-100k.pdf | 85304 | 82915 | 167546 MB/s | 7470 MB/s | 326905 MB/s | 198869 MB/s |
|
||||
| html_x_4 | 92234 | 19841 | 15194 MB/s | 23403 MB/s | 30843 MB/s | 30937 MB/s |
|
||||
| alice29.txt | 88034 | 73218 | 5936 MB/s | 2945 MB/s | 12882 MB/s | 16611 MB/s |
|
||||
| asyoulik.txt | 77503 | 66844 | 5517 MB/s | 2739 MB/s | 12735 MB/s | 14975 MB/s |
|
||||
| lcet10.txt | 234661 | 190589 | 6235 MB/s | 3099 MB/s | 14519 MB/s | 16634 MB/s |
|
||||
| plrabn12.txt | 319267 | 270828 | 5159 MB/s | 2600 MB/s | 11923 MB/s | 13382 MB/s |
|
||||
| geo.protodata | 23335 | 18278 | 21220 MB/s | 11208 MB/s | 56271 MB/s | 57961 MB/s |
|
||||
| kppkn.gtb | 69526 | 61851 | 9732 MB/s | 4556 MB/s | 18491 MB/s | 16524 MB/s |
|
||||
| alice29.txt (128B) | 80 | 81 | 6691 MB/s | 529 MB/s | 31883 MB/s | 34225 MB/s |
|
||||
| alice29.txt (1000B) | 774 | 748 | 12204 MB/s | 1943 MB/s | 48056 MB/s | 42068 MB/s |
|
||||
| alice29.txt (10000B) | 6648 | 6234 | 10044 MB/s | 2949 MB/s | 32378 MB/s | 28813 MB/s |
|
||||
| alice29.txt (20000B) | 12686 | 11584 | 7733 MB/s | 2822 MB/s | 30566 MB/s | 27315 MB/s |
|
||||
|
||||
|
||||
| Relative Perf | Snappy size | Better size | Better Speed | Better dec |
|
||||
|-----------------------|-------------|-------------|--------------|------------|
|
||||
| html | 22.31% | 13.18% | 0.48x | 0.98x |
|
||||
| urls.10K | 47.78% | 24.43% | 0.50x | 0.93x |
|
||||
| fireworks.jpeg | 99.95% | -0.05% | 0.03x | 1.15x |
|
||||
| fireworks.jpeg (200B) | 73.00% | 2.74% | 0.07x | 0.89x |
|
||||
| paper-100k.pdf | 83.30% | 2.80% | 0.07x | 0.61x |
|
||||
| html_x_4 | 22.52% | 78.49% | 0.04x | 1.00x |
|
||||
| alice29.txt | 57.88% | 16.83% | 1.54x | 1.29x |
|
||||
| asyoulik.txt | 61.91% | 13.75% | 0.50x | 1.18x |
|
||||
| lcet10.txt | 54.99% | 18.78% | 0.50x | 1.15x |
|
||||
| plrabn12.txt | 66.26% | 15.17% | 0.50x | 1.12x |
|
||||
| geo.protodata | 19.68% | 21.67% | 0.50x | 1.03x |
|
||||
| kppkn.gtb | 37.72% | 11.04% | 0.53x | 0.89x |
|
||||
| alice29.txt (128B) | 62.50% | -1.25% | 0.47x | 1.07x |
|
||||
| alice29.txt (1000B) | 77.40% | 3.36% | 0.08x | 0.88x |
|
||||
| alice29.txt (10000B) | 66.48% | 6.23% | 0.16x | 0.89x |
|
||||
| alice29.txt (20000B) | 63.43% | 8.69% | 0.29x | 0.89x |
|
||||
|
||||
Except for the mostly incompressible JPEG image compression is better and usually in the
|
||||
double digits in terms of percentage reduction over Snappy.
|
||||
|
||||
The PDF sample shows a significant slowdown compared to Snappy, as this mode tries harder
|
||||
to compress the data. Very small blocks are also not favorable for better compression, so throughput is way down.
|
||||
|
||||
This mode aims to provide better compression at the expense of performance and achieves that
|
||||
without a huge performance penalty, except on very small blocks.
|
||||
|
||||
Decompression speed suffers a little compared to the regular S2 mode,
|
||||
but still manages to be close to Snappy in spite of increased compression.
|
||||
|
||||
# Best compression mode
|
||||
|
||||
S2 offers a "best" compression mode.
|
||||
|
||||
This will compress as much as possible with little regard to CPU usage.
|
||||
|
||||
Mainly for offline compression, but where decompression speed should still
|
||||
be high and compatible with other S2 compressed data.
|
||||
|
||||
Some examples compared on 16 core CPU, amd64 assembly used:
|
||||
|
||||
```
|
||||
* enwik10
|
||||
Default... 10000000000 -> 4761467548 [47.61%]; 1.098s, 8685.6MB/s
|
||||
Better... 10000000000 -> 4219438251 [42.19%]; 1.925s, 4954.2MB/s
|
||||
Best... 10000000000 -> 3627364337 [36.27%]; 43.051s, 221.5MB/s
|
||||
|
||||
* github-june-2days-2019.json
|
||||
Default... 6273951764 -> 1043196283 [16.63%]; 431ms, 13882.3MB/s
|
||||
Better... 6273951764 -> 949146808 [15.13%]; 547ms, 10938.4MB/s
|
||||
Best... 6273951764 -> 832855506 [13.27%]; 9.455s, 632.8MB/s
|
||||
|
||||
* nyc-taxi-data-10M.csv
|
||||
Default... 3325605752 -> 1095998837 [32.96%]; 324ms, 9788.7MB/s
|
||||
Better... 3325605752 -> 954776589 [28.71%]; 491ms, 6459.4MB/s
|
||||
Best... 3325605752 -> 779098746 [23.43%]; 8.29s, 382.6MB/s
|
||||
|
||||
* 10gb.tar
|
||||
Default... 10065157632 -> 5916578242 [58.78%]; 1.028s, 9337.4MB/s
|
||||
Better... 10065157632 -> 5649207485 [56.13%]; 1.597s, 6010.6MB/s
|
||||
Best... 10065157632 -> 5208719802 [51.75%]; 32.78s, 292.8MB/
|
||||
|
||||
* consensus.db.10gb
|
||||
Default... 10737418240 -> 4562648848 [42.49%]; 882ms, 11610.0MB/s
|
||||
Better... 10737418240 -> 4542428129 [42.30%]; 1.533s, 6679.7MB/s
|
||||
Best... 10737418240 -> 4244773384 [39.53%]; 42.96s, 238.4MB/s
|
||||
```
|
||||
|
||||
Decompression speed should be around the same as using the 'better' compression mode.
|
||||
|
||||
# Snappy Compatibility
|
||||
|
||||
S2 now offers full compatibility with Snappy.
|
||||
|
||||
This means that the efficient encoders of S2 can be used to generate fully Snappy compatible output.
|
||||
|
||||
There is a [snappy](https://github.com/klauspost/compress/tree/master/snappy) package that can be used by
|
||||
simply changing imports from `github.com/golang/snappy` to `github.com/klauspost/compress/snappy`.
|
||||
This uses "better" mode for all operations.
|
||||
If you would like more control, you can use the s2 package as described below:
|
||||
|
||||
## Blocks
|
||||
|
||||
Snappy compatible blocks can be generated with the S2 encoder.
|
||||
Compression and speed is typically a bit better `MaxEncodedLen` is also smaller for smaller memory usage. Replace
|
||||
|
||||
| Snappy | S2 replacement |
|
||||
|----------------------------|-------------------------|
|
||||
| snappy.Encode(...) | s2.EncodeSnappy(...) |
|
||||
| snappy.MaxEncodedLen(...) | s2.MaxEncodedLen(...) |
|
||||
|
||||
`s2.EncodeSnappy` can be replaced with `s2.EncodeSnappyBetter` or `s2.EncodeSnappyBest` to get more efficiently compressed snappy compatible output.
|
||||
|
||||
`s2.ConcatBlocks` is compatible with snappy blocks.
|
||||
|
||||
Comparison of [`webdevdata.org-2015-01-07-subset`](https://files.klauspost.com/compress/webdevdata.org-2015-01-07-4GB-subset.7z),
|
||||
53927 files, total input size: 4,014,735,833 bytes. amd64, single goroutine used:
|
||||
|
||||
| Encoder | Size | MB/s | Reduction |
|
||||
|-----------------------|------------|--------|------------
|
||||
| snappy.Encode | 1128706759 | 725.59 | 71.89% |
|
||||
| s2.EncodeSnappy | 1093823291 | 899.16 | 72.75% |
|
||||
| s2.EncodeSnappyBetter | 1001158548 | 578.49 | 75.06% |
|
||||
| s2.EncodeSnappyBest | 944507998 | 66.00 | 76.47% |
|
||||
|
||||
## Streams
|
||||
|
||||
For streams, replace `enc = snappy.NewBufferedWriter(w)` with `enc = s2.NewWriter(w, s2.WriterSnappyCompat())`.
|
||||
All other options are available, but note that block size limit is different for snappy.
|
||||
|
||||
Comparison of different streams, AMD Ryzen 3950x, 16 cores. Size and throughput:
|
||||
|
||||
| File | snappy.NewWriter | S2 Snappy | S2 Snappy, Better | S2 Snappy, Best |
|
||||
|-----------------------------|--------------------------|---------------------------|--------------------------|-------------------------|
|
||||
| nyc-taxi-data-10M.csv | 1316042016 - 517.54MB/s | 1307003093 - 8406.29MB/s | 1174534014 - 4984.35MB/s | 1115904679 - 177.81MB/s |
|
||||
| enwik10 | 5088294643 - 433.45MB/s | 5175840939 - 8454.52MB/s | 4560784526 - 4403.10MB/s | 4340299103 - 159.71MB/s |
|
||||
| 10gb.tar | 6056946612 - 703.25MB/s | 6208571995 - 9035.75MB/s | 5741646126 - 2402.08MB/s | 5548973895 - 171.17MB/s |
|
||||
| github-june-2days-2019.json | 1525176492 - 908.11MB/s | 1476519054 - 12625.93MB/s | 1400547532 - 6163.61MB/s | 1321887137 - 200.71MB/s |
|
||||
| consensus.db.10gb | 5412897703 - 1054.38MB/s | 5354073487 - 12634.82MB/s | 5335069899 - 2472.23MB/s | 5201000954 - 166.32MB/s |
|
||||
|
||||
# Decompression
|
||||
|
||||
All decompression functions map directly to equivalent s2 functions.
|
||||
|
||||
| Snappy | S2 replacement |
|
||||
|------------------------|--------------------|
|
||||
| snappy.Decode(...) | s2.Decode(...) |
|
||||
| snappy.DecodedLen(...) | s2.DecodedLen(...) |
|
||||
| snappy.NewReader(...) | s2.NewReader(...) |
|
||||
|
||||
Features like [quick forward skipping without decompression](https://pkg.go.dev/github.com/klauspost/compress/s2#Reader.Skip)
|
||||
are also available for Snappy streams.
|
||||
|
||||
If you know you are only decompressing snappy streams, setting [`ReaderMaxBlockSize(64<<10)`](https://pkg.go.dev/github.com/klauspost/compress/s2#ReaderMaxBlockSize)
|
||||
on your Reader will reduce memory consumption.
|
||||
|
||||
# Concatenating blocks and streams.
|
||||
|
||||
Concatenating streams will concatenate the output of both without recompressing them.
|
||||
While this is inefficient in terms of compression it might be usable in certain scenarios.
|
||||
The 10 byte 'stream identifier' of the second stream can optionally be stripped, but it is not a requirement.
|
||||
|
||||
Blocks can be concatenated using the `ConcatBlocks` function.
|
||||
|
||||
Snappy blocks/streams can safely be concatenated with S2 blocks and streams.
|
||||
|
||||
# Format Extensions
|
||||
|
||||
* Frame [Stream identifier](https://github.com/google/snappy/blob/master/framing_format.txt#L68) changed from `sNaPpY` to `S2sTwO`.
|
||||
* [Framed compressed blocks](https://github.com/google/snappy/blob/master/format_description.txt) can be up to 4MB (up from 64KB).
|
||||
* Compressed blocks can have an offset of `0`, which indicates to repeat the last seen offset.
|
||||
|
||||
Repeat offsets must be encoded as a [2.2.1. Copy with 1-byte offset (01)](https://github.com/google/snappy/blob/master/format_description.txt#L89), where the offset is 0.
|
||||
|
||||
The length is specified by reading the 3-bit length specified in the tag and decode using this table:
|
||||
|
||||
| Length | Actual Length |
|
||||
|--------|----------------------|
|
||||
| 0 | 4 |
|
||||
| 1 | 5 |
|
||||
| 2 | 6 |
|
||||
| 3 | 7 |
|
||||
| 4 | 8 |
|
||||
| 5 | 8 + read 1 byte |
|
||||
| 6 | 260 + read 2 bytes |
|
||||
| 7 | 65540 + read 3 bytes |
|
||||
|
||||
This allows any repeat offset + length to be represented by 2 to 5 bytes.
|
||||
|
||||
Lengths are stored as little endian values.
|
||||
|
||||
The first copy of a block cannot be a repeat offset and the offset is not carried across blocks in streams.
|
||||
|
||||
Default streaming block size is 1MB.
|
||||
|
||||
# LICENSE
|
||||
|
||||
This code is based on the [Snappy-Go](https://github.com/golang/snappy) implementation.
|
||||
|
||||
Use of this source code is governed by a BSD-style license that can be found in the LICENSE file.
|
||||
565
vendor/github.com/klauspost/compress/s2/decode.go
сгенерированный
поставляемый
Обычный файл
565
vendor/github.com/klauspost/compress/s2/decode.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,565 @@
|
||||
// Copyright 2011 The Snappy-Go Authors. All rights reserved.
|
||||
// Copyright (c) 2019 Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package s2
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"io"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrCorrupt reports that the input is invalid.
|
||||
ErrCorrupt = errors.New("s2: corrupt input")
|
||||
// ErrCRC reports that the input failed CRC validation (streams only)
|
||||
ErrCRC = errors.New("s2: corrupt input, crc mismatch")
|
||||
// ErrTooLarge reports that the uncompressed length is too large.
|
||||
ErrTooLarge = errors.New("s2: decoded block is too large")
|
||||
// ErrUnsupported reports that the input isn't supported.
|
||||
ErrUnsupported = errors.New("s2: unsupported input")
|
||||
)
|
||||
|
||||
// DecodedLen returns the length of the decoded block.
|
||||
func DecodedLen(src []byte) (int, error) {
|
||||
v, _, err := decodedLen(src)
|
||||
return v, err
|
||||
}
|
||||
|
||||
// decodedLen returns the length of the decoded block and the number of bytes
|
||||
// that the length header occupied.
|
||||
func decodedLen(src []byte) (blockLen, headerLen int, err error) {
|
||||
v, n := binary.Uvarint(src)
|
||||
if n <= 0 || v > 0xffffffff {
|
||||
return 0, 0, ErrCorrupt
|
||||
}
|
||||
|
||||
const wordSize = 32 << (^uint(0) >> 32 & 1)
|
||||
if wordSize == 32 && v > 0x7fffffff {
|
||||
return 0, 0, ErrTooLarge
|
||||
}
|
||||
return int(v), n, nil
|
||||
}
|
||||
|
||||
const (
|
||||
decodeErrCodeCorrupt = 1
|
||||
)
|
||||
|
||||
// Decode returns the decoded form of src. The returned slice may be a sub-
|
||||
// slice of dst if dst was large enough to hold the entire decoded block.
|
||||
// Otherwise, a newly allocated slice will be returned.
|
||||
//
|
||||
// The dst and src must not overlap. It is valid to pass a nil dst.
|
||||
func Decode(dst, src []byte) ([]byte, error) {
|
||||
dLen, s, err := decodedLen(src)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if dLen <= cap(dst) {
|
||||
dst = dst[:dLen]
|
||||
} else {
|
||||
dst = make([]byte, dLen)
|
||||
}
|
||||
if s2Decode(dst, src[s:]) != 0 {
|
||||
return nil, ErrCorrupt
|
||||
}
|
||||
return dst, nil
|
||||
}
|
||||
|
||||
// NewReader returns a new Reader that decompresses from r, using the framing
|
||||
// format described at
|
||||
// https://github.com/google/snappy/blob/master/framing_format.txt with S2 changes.
|
||||
func NewReader(r io.Reader, opts ...ReaderOption) *Reader {
|
||||
nr := Reader{
|
||||
r: r,
|
||||
maxBlock: maxBlockSize,
|
||||
}
|
||||
for _, opt := range opts {
|
||||
if err := opt(&nr); err != nil {
|
||||
nr.err = err
|
||||
return &nr
|
||||
}
|
||||
}
|
||||
nr.maxBufSize = MaxEncodedLen(nr.maxBlock) + checksumSize
|
||||
if nr.lazyBuf > 0 {
|
||||
nr.buf = make([]byte, MaxEncodedLen(nr.lazyBuf)+checksumSize)
|
||||
} else {
|
||||
nr.buf = make([]byte, MaxEncodedLen(defaultBlockSize)+checksumSize)
|
||||
}
|
||||
nr.paramsOK = true
|
||||
return &nr
|
||||
}
|
||||
|
||||
// ReaderOption is an option for creating a decoder.
|
||||
type ReaderOption func(*Reader) error
|
||||
|
||||
// ReaderMaxBlockSize allows to control allocations if the stream
|
||||
// has been compressed with a smaller WriterBlockSize, or with the default 1MB.
|
||||
// Blocks must be this size or smaller to decompress,
|
||||
// otherwise the decoder will return ErrUnsupported.
|
||||
//
|
||||
// For streams compressed with Snappy this can safely be set to 64KB (64 << 10).
|
||||
//
|
||||
// Default is the maximum limit of 4MB.
|
||||
func ReaderMaxBlockSize(blockSize int) ReaderOption {
|
||||
return func(r *Reader) error {
|
||||
if blockSize > maxBlockSize || blockSize <= 0 {
|
||||
return errors.New("s2: block size too large. Must be <= 4MB and > 0")
|
||||
}
|
||||
if r.lazyBuf == 0 && blockSize < defaultBlockSize {
|
||||
r.lazyBuf = blockSize
|
||||
}
|
||||
r.maxBlock = blockSize
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// ReaderAllocBlock allows to control upfront stream allocations
|
||||
// and not allocate for frames bigger than this initially.
|
||||
// If frames bigger than this is seen a bigger buffer will be allocated.
|
||||
//
|
||||
// Default is 1MB, which is default output size.
|
||||
func ReaderAllocBlock(blockSize int) ReaderOption {
|
||||
return func(r *Reader) error {
|
||||
if blockSize > maxBlockSize || blockSize < 1024 {
|
||||
return errors.New("s2: invalid ReaderAllocBlock. Must be <= 4MB and >= 1024")
|
||||
}
|
||||
r.lazyBuf = blockSize
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// Reader is an io.Reader that can read Snappy-compressed bytes.
|
||||
type Reader struct {
|
||||
r io.Reader
|
||||
err error
|
||||
decoded []byte
|
||||
buf []byte
|
||||
// decoded[i:j] contains decoded bytes that have not yet been passed on.
|
||||
i, j int
|
||||
// maximum block size allowed.
|
||||
maxBlock int
|
||||
// maximum expected buffer size.
|
||||
maxBufSize int
|
||||
// alloc a buffer this size if > 0.
|
||||
lazyBuf int
|
||||
readHeader bool
|
||||
paramsOK bool
|
||||
snappyFrame bool
|
||||
}
|
||||
|
||||
// ensureBufferSize will ensure that the buffer can take at least n bytes.
|
||||
// If false is returned the buffer exceeds maximum allowed size.
|
||||
func (r *Reader) ensureBufferSize(n int) bool {
|
||||
if len(r.buf) >= n {
|
||||
return true
|
||||
}
|
||||
if n > r.maxBufSize {
|
||||
r.err = ErrCorrupt
|
||||
return false
|
||||
}
|
||||
// Realloc buffer.
|
||||
r.buf = make([]byte, n)
|
||||
return true
|
||||
}
|
||||
|
||||
// Reset discards any buffered data, resets all state, and switches the Snappy
|
||||
// reader to read from r. This permits reusing a Reader rather than allocating
|
||||
// a new one.
|
||||
func (r *Reader) Reset(reader io.Reader) {
|
||||
if !r.paramsOK {
|
||||
return
|
||||
}
|
||||
r.r = reader
|
||||
r.err = nil
|
||||
r.i = 0
|
||||
r.j = 0
|
||||
r.readHeader = false
|
||||
}
|
||||
|
||||
func (r *Reader) readFull(p []byte, allowEOF bool) (ok bool) {
|
||||
if _, r.err = io.ReadFull(r.r, p); r.err != nil {
|
||||
if r.err == io.ErrUnexpectedEOF || (r.err == io.EOF && !allowEOF) {
|
||||
r.err = ErrCorrupt
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// skipN will skip n bytes.
|
||||
// If the supplied reader supports seeking that is used.
|
||||
// tmp is used as a temporary buffer for reading.
|
||||
// The supplied slice does not need to be the size of the read.
|
||||
func (r *Reader) skipN(tmp []byte, n int, allowEOF bool) (ok bool) {
|
||||
if rs, ok := r.r.(io.ReadSeeker); ok {
|
||||
_, err := rs.Seek(int64(n), io.SeekCurrent)
|
||||
if err == nil {
|
||||
return true
|
||||
}
|
||||
if err == io.ErrUnexpectedEOF || (r.err == io.EOF && !allowEOF) {
|
||||
r.err = ErrCorrupt
|
||||
return false
|
||||
}
|
||||
}
|
||||
for n > 0 {
|
||||
if n < len(tmp) {
|
||||
tmp = tmp[:n]
|
||||
}
|
||||
if _, r.err = io.ReadFull(r.r, tmp); r.err != nil {
|
||||
if r.err == io.ErrUnexpectedEOF || (r.err == io.EOF && !allowEOF) {
|
||||
r.err = ErrCorrupt
|
||||
}
|
||||
return false
|
||||
}
|
||||
n -= len(tmp)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Read satisfies the io.Reader interface.
|
||||
func (r *Reader) Read(p []byte) (int, error) {
|
||||
if r.err != nil {
|
||||
return 0, r.err
|
||||
}
|
||||
for {
|
||||
if r.i < r.j {
|
||||
n := copy(p, r.decoded[r.i:r.j])
|
||||
r.i += n
|
||||
return n, nil
|
||||
}
|
||||
if !r.readFull(r.buf[:4], true) {
|
||||
return 0, r.err
|
||||
}
|
||||
chunkType := r.buf[0]
|
||||
if !r.readHeader {
|
||||
if chunkType != chunkTypeStreamIdentifier {
|
||||
r.err = ErrCorrupt
|
||||
return 0, r.err
|
||||
}
|
||||
r.readHeader = true
|
||||
}
|
||||
chunkLen := int(r.buf[1]) | int(r.buf[2])<<8 | int(r.buf[3])<<16
|
||||
|
||||
// The chunk types are specified at
|
||||
// https://github.com/google/snappy/blob/master/framing_format.txt
|
||||
switch chunkType {
|
||||
case chunkTypeCompressedData:
|
||||
// Section 4.2. Compressed data (chunk type 0x00).
|
||||
if chunkLen < checksumSize {
|
||||
r.err = ErrCorrupt
|
||||
return 0, r.err
|
||||
}
|
||||
if !r.ensureBufferSize(chunkLen) {
|
||||
if r.err == nil {
|
||||
r.err = ErrUnsupported
|
||||
}
|
||||
return 0, r.err
|
||||
}
|
||||
buf := r.buf[:chunkLen]
|
||||
if !r.readFull(buf, false) {
|
||||
return 0, r.err
|
||||
}
|
||||
checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24
|
||||
buf = buf[checksumSize:]
|
||||
|
||||
n, err := DecodedLen(buf)
|
||||
if err != nil {
|
||||
r.err = err
|
||||
return 0, r.err
|
||||
}
|
||||
if r.snappyFrame && n > maxSnappyBlockSize {
|
||||
r.err = ErrCorrupt
|
||||
return 0, r.err
|
||||
}
|
||||
|
||||
if n > len(r.decoded) {
|
||||
if n > r.maxBlock {
|
||||
r.err = ErrCorrupt
|
||||
return 0, r.err
|
||||
}
|
||||
r.decoded = make([]byte, n)
|
||||
}
|
||||
if _, err := Decode(r.decoded, buf); err != nil {
|
||||
r.err = err
|
||||
return 0, r.err
|
||||
}
|
||||
if crc(r.decoded[:n]) != checksum {
|
||||
r.err = ErrCRC
|
||||
return 0, r.err
|
||||
}
|
||||
r.i, r.j = 0, n
|
||||
continue
|
||||
|
||||
case chunkTypeUncompressedData:
|
||||
// Section 4.3. Uncompressed data (chunk type 0x01).
|
||||
if chunkLen < checksumSize {
|
||||
r.err = ErrCorrupt
|
||||
return 0, r.err
|
||||
}
|
||||
if !r.ensureBufferSize(chunkLen) {
|
||||
if r.err == nil {
|
||||
r.err = ErrUnsupported
|
||||
}
|
||||
return 0, r.err
|
||||
}
|
||||
buf := r.buf[:checksumSize]
|
||||
if !r.readFull(buf, false) {
|
||||
return 0, r.err
|
||||
}
|
||||
checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24
|
||||
// Read directly into r.decoded instead of via r.buf.
|
||||
n := chunkLen - checksumSize
|
||||
if r.snappyFrame && n > maxSnappyBlockSize {
|
||||
r.err = ErrCorrupt
|
||||
return 0, r.err
|
||||
}
|
||||
if n > len(r.decoded) {
|
||||
if n > r.maxBlock {
|
||||
r.err = ErrCorrupt
|
||||
return 0, r.err
|
||||
}
|
||||
r.decoded = make([]byte, n)
|
||||
}
|
||||
if !r.readFull(r.decoded[:n], false) {
|
||||
return 0, r.err
|
||||
}
|
||||
if crc(r.decoded[:n]) != checksum {
|
||||
r.err = ErrCRC
|
||||
return 0, r.err
|
||||
}
|
||||
r.i, r.j = 0, n
|
||||
continue
|
||||
|
||||
case chunkTypeStreamIdentifier:
|
||||
// Section 4.1. Stream identifier (chunk type 0xff).
|
||||
if chunkLen != len(magicBody) {
|
||||
r.err = ErrCorrupt
|
||||
return 0, r.err
|
||||
}
|
||||
if !r.readFull(r.buf[:len(magicBody)], false) {
|
||||
return 0, r.err
|
||||
}
|
||||
if string(r.buf[:len(magicBody)]) != magicBody {
|
||||
if string(r.buf[:len(magicBody)]) != magicBodySnappy {
|
||||
r.err = ErrCorrupt
|
||||
return 0, r.err
|
||||
} else {
|
||||
r.snappyFrame = true
|
||||
}
|
||||
} else {
|
||||
r.snappyFrame = false
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if chunkType <= 0x7f {
|
||||
// Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f).
|
||||
r.err = ErrUnsupported
|
||||
return 0, r.err
|
||||
}
|
||||
// Section 4.4 Padding (chunk type 0xfe).
|
||||
// Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd).
|
||||
if chunkLen > maxBlockSize {
|
||||
r.err = ErrUnsupported
|
||||
return 0, r.err
|
||||
}
|
||||
|
||||
if !r.skipN(r.buf, chunkLen, false) {
|
||||
return 0, r.err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Skip will skip n bytes forward in the decompressed output.
|
||||
// For larger skips this consumes less CPU and is faster than reading output and discarding it.
|
||||
// CRC is not checked on skipped blocks.
|
||||
// io.ErrUnexpectedEOF is returned if the stream ends before all bytes have been skipped.
|
||||
// If a decoding error is encountered subsequent calls to Read will also fail.
|
||||
func (r *Reader) Skip(n int64) error {
|
||||
if n < 0 {
|
||||
return errors.New("attempted negative skip")
|
||||
}
|
||||
if r.err != nil {
|
||||
return r.err
|
||||
}
|
||||
|
||||
for n > 0 {
|
||||
if r.i < r.j {
|
||||
// Skip in buffer.
|
||||
// decoded[i:j] contains decoded bytes that have not yet been passed on.
|
||||
left := int64(r.j - r.i)
|
||||
if left >= n {
|
||||
r.i += int(n)
|
||||
return nil
|
||||
}
|
||||
n -= int64(r.j - r.i)
|
||||
r.i, r.j = 0, 0
|
||||
}
|
||||
|
||||
// Buffer empty; read blocks until we have content.
|
||||
if !r.readFull(r.buf[:4], true) {
|
||||
if r.err == io.EOF {
|
||||
r.err = io.ErrUnexpectedEOF
|
||||
}
|
||||
return r.err
|
||||
}
|
||||
chunkType := r.buf[0]
|
||||
if !r.readHeader {
|
||||
if chunkType != chunkTypeStreamIdentifier {
|
||||
r.err = ErrCorrupt
|
||||
return r.err
|
||||
}
|
||||
r.readHeader = true
|
||||
}
|
||||
chunkLen := int(r.buf[1]) | int(r.buf[2])<<8 | int(r.buf[3])<<16
|
||||
|
||||
// The chunk types are specified at
|
||||
// https://github.com/google/snappy/blob/master/framing_format.txt
|
||||
switch chunkType {
|
||||
case chunkTypeCompressedData:
|
||||
// Section 4.2. Compressed data (chunk type 0x00).
|
||||
if chunkLen < checksumSize {
|
||||
r.err = ErrCorrupt
|
||||
return r.err
|
||||
}
|
||||
if !r.ensureBufferSize(chunkLen) {
|
||||
if r.err == nil {
|
||||
r.err = ErrUnsupported
|
||||
}
|
||||
return r.err
|
||||
}
|
||||
buf := r.buf[:chunkLen]
|
||||
if !r.readFull(buf, false) {
|
||||
return r.err
|
||||
}
|
||||
checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24
|
||||
buf = buf[checksumSize:]
|
||||
|
||||
dLen, err := DecodedLen(buf)
|
||||
if err != nil {
|
||||
r.err = err
|
||||
return r.err
|
||||
}
|
||||
if dLen > r.maxBlock {
|
||||
r.err = ErrCorrupt
|
||||
return r.err
|
||||
}
|
||||
// Check if destination is within this block
|
||||
if int64(dLen) > n {
|
||||
if len(r.decoded) < dLen {
|
||||
r.decoded = make([]byte, dLen)
|
||||
}
|
||||
if _, err := Decode(r.decoded, buf); err != nil {
|
||||
r.err = err
|
||||
return r.err
|
||||
}
|
||||
if crc(r.decoded[:dLen]) != checksum {
|
||||
r.err = ErrCorrupt
|
||||
return r.err
|
||||
}
|
||||
} else {
|
||||
// Skip block completely
|
||||
n -= int64(dLen)
|
||||
dLen = 0
|
||||
}
|
||||
r.i, r.j = 0, dLen
|
||||
continue
|
||||
case chunkTypeUncompressedData:
|
||||
// Section 4.3. Uncompressed data (chunk type 0x01).
|
||||
if chunkLen < checksumSize {
|
||||
r.err = ErrCorrupt
|
||||
return r.err
|
||||
}
|
||||
if !r.ensureBufferSize(chunkLen) {
|
||||
if r.err != nil {
|
||||
r.err = ErrUnsupported
|
||||
}
|
||||
return r.err
|
||||
}
|
||||
buf := r.buf[:checksumSize]
|
||||
if !r.readFull(buf, false) {
|
||||
return r.err
|
||||
}
|
||||
checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24
|
||||
// Read directly into r.decoded instead of via r.buf.
|
||||
n2 := chunkLen - checksumSize
|
||||
if n2 > len(r.decoded) {
|
||||
if n2 > r.maxBlock {
|
||||
r.err = ErrCorrupt
|
||||
return r.err
|
||||
}
|
||||
r.decoded = make([]byte, n2)
|
||||
}
|
||||
if !r.readFull(r.decoded[:n2], false) {
|
||||
return r.err
|
||||
}
|
||||
if int64(n2) < n {
|
||||
if crc(r.decoded[:n2]) != checksum {
|
||||
r.err = ErrCorrupt
|
||||
return r.err
|
||||
}
|
||||
}
|
||||
r.i, r.j = 0, n2
|
||||
continue
|
||||
case chunkTypeStreamIdentifier:
|
||||
// Section 4.1. Stream identifier (chunk type 0xff).
|
||||
if chunkLen != len(magicBody) {
|
||||
r.err = ErrCorrupt
|
||||
return r.err
|
||||
}
|
||||
if !r.readFull(r.buf[:len(magicBody)], false) {
|
||||
return r.err
|
||||
}
|
||||
if string(r.buf[:len(magicBody)]) != magicBody {
|
||||
if string(r.buf[:len(magicBody)]) != magicBodySnappy {
|
||||
r.err = ErrCorrupt
|
||||
return r.err
|
||||
}
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if chunkType <= 0x7f {
|
||||
// Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f).
|
||||
r.err = ErrUnsupported
|
||||
return r.err
|
||||
}
|
||||
if chunkLen > maxBlockSize {
|
||||
r.err = ErrUnsupported
|
||||
return r.err
|
||||
}
|
||||
// Section 4.4 Padding (chunk type 0xfe).
|
||||
// Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd).
|
||||
if !r.skipN(r.buf, chunkLen, false) {
|
||||
return r.err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReadByte satisfies the io.ByteReader interface.
|
||||
func (r *Reader) ReadByte() (byte, error) {
|
||||
if r.err != nil {
|
||||
return 0, r.err
|
||||
}
|
||||
if r.i < r.j {
|
||||
c := r.decoded[r.i]
|
||||
r.i++
|
||||
return c, nil
|
||||
}
|
||||
var tmp [1]byte
|
||||
for i := 0; i < 10; i++ {
|
||||
n, err := r.Read(tmp[:])
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if n == 1 {
|
||||
return tmp[0], nil
|
||||
}
|
||||
}
|
||||
return 0, io.ErrNoProgress
|
||||
}
|
||||
568
vendor/github.com/klauspost/compress/s2/decode_amd64.s
сгенерированный
поставляемый
Обычный файл
568
vendor/github.com/klauspost/compress/s2/decode_amd64.s
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,568 @@
|
||||
// Copyright 2016 The Go Authors. All rights reserved.
|
||||
// Copyright (c) 2019 Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build !appengine
|
||||
// +build gc
|
||||
// +build !noasm
|
||||
|
||||
#include "textflag.h"
|
||||
|
||||
#define R_TMP0 AX
|
||||
#define R_TMP1 BX
|
||||
#define R_LEN CX
|
||||
#define R_OFF DX
|
||||
#define R_SRC SI
|
||||
#define R_DST DI
|
||||
#define R_DBASE R8
|
||||
#define R_DLEN R9
|
||||
#define R_DEND R10
|
||||
#define R_SBASE R11
|
||||
#define R_SLEN R12
|
||||
#define R_SEND R13
|
||||
#define R_TMP2 R14
|
||||
#define R_TMP3 R15
|
||||
|
||||
// The asm code generally follows the pure Go code in decode_other.go, except
|
||||
// where marked with a "!!!".
|
||||
|
||||
// func decode(dst, src []byte) int
|
||||
//
|
||||
// All local variables fit into registers. The non-zero stack size is only to
|
||||
// spill registers and push args when issuing a CALL. The register allocation:
|
||||
// - R_TMP0 scratch
|
||||
// - R_TMP1 scratch
|
||||
// - R_LEN length or x (shared)
|
||||
// - R_OFF offset
|
||||
// - R_SRC &src[s]
|
||||
// - R_DST &dst[d]
|
||||
// + R_DBASE dst_base
|
||||
// + R_DLEN dst_len
|
||||
// + R_DEND dst_base + dst_len
|
||||
// + R_SBASE src_base
|
||||
// + R_SLEN src_len
|
||||
// + R_SEND src_base + src_len
|
||||
// - R_TMP2 used by doCopy
|
||||
// - R_TMP3 used by doCopy
|
||||
//
|
||||
// The registers R_DBASE-R_SEND (marked with a "+") are set at the start of the
|
||||
// function, and after a CALL returns, and are not otherwise modified.
|
||||
//
|
||||
// The d variable is implicitly R_DST - R_DBASE, and len(dst)-d is R_DEND - R_DST.
|
||||
// The s variable is implicitly R_SRC - R_SBASE, and len(src)-s is R_SEND - R_SRC.
|
||||
TEXT ·s2Decode(SB), NOSPLIT, $48-56
|
||||
// Initialize R_SRC, R_DST and R_DBASE-R_SEND.
|
||||
MOVQ dst_base+0(FP), R_DBASE
|
||||
MOVQ dst_len+8(FP), R_DLEN
|
||||
MOVQ R_DBASE, R_DST
|
||||
MOVQ R_DBASE, R_DEND
|
||||
ADDQ R_DLEN, R_DEND
|
||||
MOVQ src_base+24(FP), R_SBASE
|
||||
MOVQ src_len+32(FP), R_SLEN
|
||||
MOVQ R_SBASE, R_SRC
|
||||
MOVQ R_SBASE, R_SEND
|
||||
ADDQ R_SLEN, R_SEND
|
||||
XORQ R_OFF, R_OFF
|
||||
|
||||
loop:
|
||||
// for s < len(src)
|
||||
CMPQ R_SRC, R_SEND
|
||||
JEQ end
|
||||
|
||||
// R_LEN = uint32(src[s])
|
||||
//
|
||||
// switch src[s] & 0x03
|
||||
MOVBLZX (R_SRC), R_LEN
|
||||
MOVL R_LEN, R_TMP1
|
||||
ANDL $3, R_TMP1
|
||||
CMPL R_TMP1, $1
|
||||
JAE tagCopy
|
||||
|
||||
// ----------------------------------------
|
||||
// The code below handles literal tags.
|
||||
|
||||
// case tagLiteral:
|
||||
// x := uint32(src[s] >> 2)
|
||||
// switch
|
||||
SHRL $2, R_LEN
|
||||
CMPL R_LEN, $60
|
||||
JAE tagLit60Plus
|
||||
|
||||
// case x < 60:
|
||||
// s++
|
||||
INCQ R_SRC
|
||||
|
||||
doLit:
|
||||
// This is the end of the inner "switch", when we have a literal tag.
|
||||
//
|
||||
// We assume that R_LEN == x and x fits in a uint32, where x is the variable
|
||||
// used in the pure Go decode_other.go code.
|
||||
|
||||
// length = int(x) + 1
|
||||
//
|
||||
// Unlike the pure Go code, we don't need to check if length <= 0 because
|
||||
// R_LEN can hold 64 bits, so the increment cannot overflow.
|
||||
INCQ R_LEN
|
||||
|
||||
// Prepare to check if copying length bytes will run past the end of dst or
|
||||
// src.
|
||||
//
|
||||
// R_TMP0 = len(dst) - d
|
||||
// R_TMP1 = len(src) - s
|
||||
MOVQ R_DEND, R_TMP0
|
||||
SUBQ R_DST, R_TMP0
|
||||
MOVQ R_SEND, R_TMP1
|
||||
SUBQ R_SRC, R_TMP1
|
||||
|
||||
// !!! Try a faster technique for short (16 or fewer bytes) copies.
|
||||
//
|
||||
// if length > 16 || len(dst)-d < 16 || len(src)-s < 16 {
|
||||
// goto callMemmove // Fall back on calling runtime·memmove.
|
||||
// }
|
||||
//
|
||||
// The C++ snappy code calls this TryFastAppend. It also checks len(src)-s
|
||||
// against 21 instead of 16, because it cannot assume that all of its input
|
||||
// is contiguous in memory and so it needs to leave enough source bytes to
|
||||
// read the next tag without refilling buffers, but Go's Decode assumes
|
||||
// contiguousness (the src argument is a []byte).
|
||||
CMPQ R_LEN, $16
|
||||
JGT callMemmove
|
||||
CMPQ R_TMP0, $16
|
||||
JLT callMemmove
|
||||
CMPQ R_TMP1, $16
|
||||
JLT callMemmove
|
||||
|
||||
// !!! Implement the copy from src to dst as a 16-byte load and store.
|
||||
// (Decode's documentation says that dst and src must not overlap.)
|
||||
//
|
||||
// This always copies 16 bytes, instead of only length bytes, but that's
|
||||
// OK. If the input is a valid Snappy encoding then subsequent iterations
|
||||
// will fix up the overrun. Otherwise, Decode returns a nil []byte (and a
|
||||
// non-nil error), so the overrun will be ignored.
|
||||
//
|
||||
// Note that on amd64, it is legal and cheap to issue unaligned 8-byte or
|
||||
// 16-byte loads and stores. This technique probably wouldn't be as
|
||||
// effective on architectures that are fussier about alignment.
|
||||
MOVOU 0(R_SRC), X0
|
||||
MOVOU X0, 0(R_DST)
|
||||
|
||||
// d += length
|
||||
// s += length
|
||||
ADDQ R_LEN, R_DST
|
||||
ADDQ R_LEN, R_SRC
|
||||
JMP loop
|
||||
|
||||
callMemmove:
|
||||
// if length > len(dst)-d || length > len(src)-s { etc }
|
||||
CMPQ R_LEN, R_TMP0
|
||||
JGT errCorrupt
|
||||
CMPQ R_LEN, R_TMP1
|
||||
JGT errCorrupt
|
||||
|
||||
// copy(dst[d:], src[s:s+length])
|
||||
//
|
||||
// This means calling runtime·memmove(&dst[d], &src[s], length), so we push
|
||||
// R_DST, R_SRC and R_LEN as arguments. Coincidentally, we also need to spill those
|
||||
// three registers to the stack, to save local variables across the CALL.
|
||||
MOVQ R_DST, 0(SP)
|
||||
MOVQ R_SRC, 8(SP)
|
||||
MOVQ R_LEN, 16(SP)
|
||||
MOVQ R_DST, 24(SP)
|
||||
MOVQ R_SRC, 32(SP)
|
||||
MOVQ R_LEN, 40(SP)
|
||||
MOVQ R_OFF, 48(SP)
|
||||
CALL runtime·memmove(SB)
|
||||
|
||||
// Restore local variables: unspill registers from the stack and
|
||||
// re-calculate R_DBASE-R_SEND.
|
||||
MOVQ 24(SP), R_DST
|
||||
MOVQ 32(SP), R_SRC
|
||||
MOVQ 40(SP), R_LEN
|
||||
MOVQ 48(SP), R_OFF
|
||||
MOVQ dst_base+0(FP), R_DBASE
|
||||
MOVQ dst_len+8(FP), R_DLEN
|
||||
MOVQ R_DBASE, R_DEND
|
||||
ADDQ R_DLEN, R_DEND
|
||||
MOVQ src_base+24(FP), R_SBASE
|
||||
MOVQ src_len+32(FP), R_SLEN
|
||||
MOVQ R_SBASE, R_SEND
|
||||
ADDQ R_SLEN, R_SEND
|
||||
|
||||
// d += length
|
||||
// s += length
|
||||
ADDQ R_LEN, R_DST
|
||||
ADDQ R_LEN, R_SRC
|
||||
JMP loop
|
||||
|
||||
tagLit60Plus:
|
||||
// !!! This fragment does the
|
||||
//
|
||||
// s += x - 58; if uint(s) > uint(len(src)) { etc }
|
||||
//
|
||||
// checks. In the asm version, we code it once instead of once per switch case.
|
||||
ADDQ R_LEN, R_SRC
|
||||
SUBQ $58, R_SRC
|
||||
CMPQ R_SRC, R_SEND
|
||||
JA errCorrupt
|
||||
|
||||
// case x == 60:
|
||||
CMPL R_LEN, $61
|
||||
JEQ tagLit61
|
||||
JA tagLit62Plus
|
||||
|
||||
// x = uint32(src[s-1])
|
||||
MOVBLZX -1(R_SRC), R_LEN
|
||||
JMP doLit
|
||||
|
||||
tagLit61:
|
||||
// case x == 61:
|
||||
// x = uint32(src[s-2]) | uint32(src[s-1])<<8
|
||||
MOVWLZX -2(R_SRC), R_LEN
|
||||
JMP doLit
|
||||
|
||||
tagLit62Plus:
|
||||
CMPL R_LEN, $62
|
||||
JA tagLit63
|
||||
|
||||
// case x == 62:
|
||||
// x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16
|
||||
// We read one byte, safe to read one back, since we are just reading tag.
|
||||
// x = binary.LittleEndian.Uint32(src[s-1:]) >> 8
|
||||
MOVL -4(R_SRC), R_LEN
|
||||
SHRL $8, R_LEN
|
||||
JMP doLit
|
||||
|
||||
tagLit63:
|
||||
// case x == 63:
|
||||
// x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24
|
||||
MOVL -4(R_SRC), R_LEN
|
||||
JMP doLit
|
||||
|
||||
// The code above handles literal tags.
|
||||
// ----------------------------------------
|
||||
// The code below handles copy tags.
|
||||
|
||||
tagCopy4:
|
||||
// case tagCopy4:
|
||||
// s += 5
|
||||
ADDQ $5, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
CMPQ R_SRC, R_SEND
|
||||
JA errCorrupt
|
||||
|
||||
// length = 1 + int(src[s-5])>>2
|
||||
SHRQ $2, R_LEN
|
||||
INCQ R_LEN
|
||||
|
||||
// offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24)
|
||||
MOVLQZX -4(R_SRC), R_OFF
|
||||
JMP doCopy
|
||||
|
||||
tagCopy2:
|
||||
// case tagCopy2:
|
||||
// s += 3
|
||||
ADDQ $3, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
CMPQ R_SRC, R_SEND
|
||||
JA errCorrupt
|
||||
|
||||
// length = 1 + int(src[s-3])>>2
|
||||
SHRQ $2, R_LEN
|
||||
INCQ R_LEN
|
||||
|
||||
// offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8)
|
||||
MOVWQZX -2(R_SRC), R_OFF
|
||||
JMP doCopy
|
||||
|
||||
tagCopy:
|
||||
// We have a copy tag. We assume that:
|
||||
// - R_TMP1 == src[s] & 0x03
|
||||
// - R_LEN == src[s]
|
||||
CMPQ R_TMP1, $2
|
||||
JEQ tagCopy2
|
||||
JA tagCopy4
|
||||
|
||||
// case tagCopy1:
|
||||
// s += 2
|
||||
ADDQ $2, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
CMPQ R_SRC, R_SEND
|
||||
JA errCorrupt
|
||||
|
||||
// offset = int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1]))
|
||||
// length = 4 + int(src[s-2])>>2&0x7
|
||||
MOVBQZX -1(R_SRC), R_TMP1
|
||||
MOVQ R_LEN, R_TMP0
|
||||
SHRQ $2, R_LEN
|
||||
ANDQ $0xe0, R_TMP0
|
||||
ANDQ $7, R_LEN
|
||||
SHLQ $3, R_TMP0
|
||||
ADDQ $4, R_LEN
|
||||
ORQ R_TMP1, R_TMP0
|
||||
|
||||
// check if repeat code, ZF set by ORQ.
|
||||
JZ repeatCode
|
||||
|
||||
// This is a regular copy, transfer our temporary value to R_OFF (length)
|
||||
MOVQ R_TMP0, R_OFF
|
||||
JMP doCopy
|
||||
|
||||
// This is a repeat code.
|
||||
repeatCode:
|
||||
// If length < 9, reuse last offset, with the length already calculated.
|
||||
CMPQ R_LEN, $9
|
||||
JL doCopyRepeat
|
||||
|
||||
// Read additional bytes for length.
|
||||
JE repeatLen1
|
||||
|
||||
// Rare, so the extra branch shouldn't hurt too much.
|
||||
CMPQ R_LEN, $10
|
||||
JE repeatLen2
|
||||
JMP repeatLen3
|
||||
|
||||
// Read repeat lengths.
|
||||
repeatLen1:
|
||||
// s ++
|
||||
ADDQ $1, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
CMPQ R_SRC, R_SEND
|
||||
JA errCorrupt
|
||||
|
||||
// length = src[s-1] + 8
|
||||
MOVBQZX -1(R_SRC), R_LEN
|
||||
ADDL $8, R_LEN
|
||||
JMP doCopyRepeat
|
||||
|
||||
repeatLen2:
|
||||
// s +=2
|
||||
ADDQ $2, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
CMPQ R_SRC, R_SEND
|
||||
JA errCorrupt
|
||||
|
||||
// length = uint32(src[s-2]) | (uint32(src[s-1])<<8) + (1 << 8)
|
||||
MOVWQZX -2(R_SRC), R_LEN
|
||||
ADDL $260, R_LEN
|
||||
JMP doCopyRepeat
|
||||
|
||||
repeatLen3:
|
||||
// s +=3
|
||||
ADDQ $3, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
CMPQ R_SRC, R_SEND
|
||||
JA errCorrupt
|
||||
|
||||
// length = uint32(src[s-3]) | (uint32(src[s-2])<<8) | (uint32(src[s-1])<<16) + (1 << 16)
|
||||
// Read one byte further back (just part of the tag, shifted out)
|
||||
MOVL -4(R_SRC), R_LEN
|
||||
SHRL $8, R_LEN
|
||||
ADDL $65540, R_LEN
|
||||
JMP doCopyRepeat
|
||||
|
||||
doCopy:
|
||||
// This is the end of the outer "switch", when we have a copy tag.
|
||||
//
|
||||
// We assume that:
|
||||
// - R_LEN == length && R_LEN > 0
|
||||
// - R_OFF == offset
|
||||
|
||||
// if d < offset { etc }
|
||||
MOVQ R_DST, R_TMP1
|
||||
SUBQ R_DBASE, R_TMP1
|
||||
CMPQ R_TMP1, R_OFF
|
||||
JLT errCorrupt
|
||||
|
||||
// Repeat values can skip the test above, since any offset > 0 will be in dst.
|
||||
doCopyRepeat:
|
||||
// if offset <= 0 { etc }
|
||||
CMPQ R_OFF, $0
|
||||
JLE errCorrupt
|
||||
|
||||
// if length > len(dst)-d { etc }
|
||||
MOVQ R_DEND, R_TMP1
|
||||
SUBQ R_DST, R_TMP1
|
||||
CMPQ R_LEN, R_TMP1
|
||||
JGT errCorrupt
|
||||
|
||||
// forwardCopy(dst[d:d+length], dst[d-offset:]); d += length
|
||||
//
|
||||
// Set:
|
||||
// - R_TMP2 = len(dst)-d
|
||||
// - R_TMP3 = &dst[d-offset]
|
||||
MOVQ R_DEND, R_TMP2
|
||||
SUBQ R_DST, R_TMP2
|
||||
MOVQ R_DST, R_TMP3
|
||||
SUBQ R_OFF, R_TMP3
|
||||
|
||||
// !!! Try a faster technique for short (16 or fewer bytes) forward copies.
|
||||
//
|
||||
// First, try using two 8-byte load/stores, similar to the doLit technique
|
||||
// above. Even if dst[d:d+length] and dst[d-offset:] can overlap, this is
|
||||
// still OK if offset >= 8. Note that this has to be two 8-byte load/stores
|
||||
// and not one 16-byte load/store, and the first store has to be before the
|
||||
// second load, due to the overlap if offset is in the range [8, 16).
|
||||
//
|
||||
// if length > 16 || offset < 8 || len(dst)-d < 16 {
|
||||
// goto slowForwardCopy
|
||||
// }
|
||||
// copy 16 bytes
|
||||
// d += length
|
||||
CMPQ R_LEN, $16
|
||||
JGT slowForwardCopy
|
||||
CMPQ R_OFF, $8
|
||||
JLT slowForwardCopy
|
||||
CMPQ R_TMP2, $16
|
||||
JLT slowForwardCopy
|
||||
MOVQ 0(R_TMP3), R_TMP0
|
||||
MOVQ R_TMP0, 0(R_DST)
|
||||
MOVQ 8(R_TMP3), R_TMP1
|
||||
MOVQ R_TMP1, 8(R_DST)
|
||||
ADDQ R_LEN, R_DST
|
||||
JMP loop
|
||||
|
||||
slowForwardCopy:
|
||||
// !!! If the forward copy is longer than 16 bytes, or if offset < 8, we
|
||||
// can still try 8-byte load stores, provided we can overrun up to 10 extra
|
||||
// bytes. As above, the overrun will be fixed up by subsequent iterations
|
||||
// of the outermost loop.
|
||||
//
|
||||
// The C++ snappy code calls this technique IncrementalCopyFastPath. Its
|
||||
// commentary says:
|
||||
//
|
||||
// ----
|
||||
//
|
||||
// The main part of this loop is a simple copy of eight bytes at a time
|
||||
// until we've copied (at least) the requested amount of bytes. However,
|
||||
// if d and d-offset are less than eight bytes apart (indicating a
|
||||
// repeating pattern of length < 8), we first need to expand the pattern in
|
||||
// order to get the correct results. For instance, if the buffer looks like
|
||||
// this, with the eight-byte <d-offset> and <d> patterns marked as
|
||||
// intervals:
|
||||
//
|
||||
// abxxxxxxxxxxxx
|
||||
// [------] d-offset
|
||||
// [------] d
|
||||
//
|
||||
// a single eight-byte copy from <d-offset> to <d> will repeat the pattern
|
||||
// once, after which we can move <d> two bytes without moving <d-offset>:
|
||||
//
|
||||
// ababxxxxxxxxxx
|
||||
// [------] d-offset
|
||||
// [------] d
|
||||
//
|
||||
// and repeat the exercise until the two no longer overlap.
|
||||
//
|
||||
// This allows us to do very well in the special case of one single byte
|
||||
// repeated many times, without taking a big hit for more general cases.
|
||||
//
|
||||
// The worst case of extra writing past the end of the match occurs when
|
||||
// offset == 1 and length == 1; the last copy will read from byte positions
|
||||
// [0..7] and write to [4..11], whereas it was only supposed to write to
|
||||
// position 1. Thus, ten excess bytes.
|
||||
//
|
||||
// ----
|
||||
//
|
||||
// That "10 byte overrun" worst case is confirmed by Go's
|
||||
// TestSlowForwardCopyOverrun, which also tests the fixUpSlowForwardCopy
|
||||
// and finishSlowForwardCopy algorithm.
|
||||
//
|
||||
// if length > len(dst)-d-10 {
|
||||
// goto verySlowForwardCopy
|
||||
// }
|
||||
SUBQ $10, R_TMP2
|
||||
CMPQ R_LEN, R_TMP2
|
||||
JGT verySlowForwardCopy
|
||||
|
||||
// We want to keep the offset, so we use R_TMP2 from here.
|
||||
MOVQ R_OFF, R_TMP2
|
||||
|
||||
makeOffsetAtLeast8:
|
||||
// !!! As above, expand the pattern so that offset >= 8 and we can use
|
||||
// 8-byte load/stores.
|
||||
//
|
||||
// for offset < 8 {
|
||||
// copy 8 bytes from dst[d-offset:] to dst[d:]
|
||||
// length -= offset
|
||||
// d += offset
|
||||
// offset += offset
|
||||
// // The two previous lines together means that d-offset, and therefore
|
||||
// // R_TMP3, is unchanged.
|
||||
// }
|
||||
CMPQ R_TMP2, $8
|
||||
JGE fixUpSlowForwardCopy
|
||||
MOVQ (R_TMP3), R_TMP1
|
||||
MOVQ R_TMP1, (R_DST)
|
||||
SUBQ R_TMP2, R_LEN
|
||||
ADDQ R_TMP2, R_DST
|
||||
ADDQ R_TMP2, R_TMP2
|
||||
JMP makeOffsetAtLeast8
|
||||
|
||||
fixUpSlowForwardCopy:
|
||||
// !!! Add length (which might be negative now) to d (implied by R_DST being
|
||||
// &dst[d]) so that d ends up at the right place when we jump back to the
|
||||
// top of the loop. Before we do that, though, we save R_DST to R_TMP0 so that, if
|
||||
// length is positive, copying the remaining length bytes will write to the
|
||||
// right place.
|
||||
MOVQ R_DST, R_TMP0
|
||||
ADDQ R_LEN, R_DST
|
||||
|
||||
finishSlowForwardCopy:
|
||||
// !!! Repeat 8-byte load/stores until length <= 0. Ending with a negative
|
||||
// length means that we overrun, but as above, that will be fixed up by
|
||||
// subsequent iterations of the outermost loop.
|
||||
CMPQ R_LEN, $0
|
||||
JLE loop
|
||||
MOVQ (R_TMP3), R_TMP1
|
||||
MOVQ R_TMP1, (R_TMP0)
|
||||
ADDQ $8, R_TMP3
|
||||
ADDQ $8, R_TMP0
|
||||
SUBQ $8, R_LEN
|
||||
JMP finishSlowForwardCopy
|
||||
|
||||
verySlowForwardCopy:
|
||||
// verySlowForwardCopy is a simple implementation of forward copy. In C
|
||||
// parlance, this is a do/while loop instead of a while loop, since we know
|
||||
// that length > 0. In Go syntax:
|
||||
//
|
||||
// for {
|
||||
// dst[d] = dst[d - offset]
|
||||
// d++
|
||||
// length--
|
||||
// if length == 0 {
|
||||
// break
|
||||
// }
|
||||
// }
|
||||
MOVB (R_TMP3), R_TMP1
|
||||
MOVB R_TMP1, (R_DST)
|
||||
INCQ R_TMP3
|
||||
INCQ R_DST
|
||||
DECQ R_LEN
|
||||
JNZ verySlowForwardCopy
|
||||
JMP loop
|
||||
|
||||
// The code above handles copy tags.
|
||||
// ----------------------------------------
|
||||
|
||||
end:
|
||||
// This is the end of the "for s < len(src)".
|
||||
//
|
||||
// if d != len(dst) { etc }
|
||||
CMPQ R_DST, R_DEND
|
||||
JNE errCorrupt
|
||||
|
||||
// return 0
|
||||
MOVQ $0, ret+48(FP)
|
||||
RET
|
||||
|
||||
errCorrupt:
|
||||
// return decodeErrCodeCorrupt
|
||||
MOVQ $1, ret+48(FP)
|
||||
RET
|
||||
574
vendor/github.com/klauspost/compress/s2/decode_arm64.s
сгенерированный
поставляемый
Обычный файл
574
vendor/github.com/klauspost/compress/s2/decode_arm64.s
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,574 @@
|
||||
// Copyright 2020 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build !appengine
|
||||
// +build gc
|
||||
// +build !noasm
|
||||
|
||||
#include "textflag.h"
|
||||
|
||||
#define R_TMP0 R2
|
||||
#define R_TMP1 R3
|
||||
#define R_LEN R4
|
||||
#define R_OFF R5
|
||||
#define R_SRC R6
|
||||
#define R_DST R7
|
||||
#define R_DBASE R8
|
||||
#define R_DLEN R9
|
||||
#define R_DEND R10
|
||||
#define R_SBASE R11
|
||||
#define R_SLEN R12
|
||||
#define R_SEND R13
|
||||
#define R_TMP2 R14
|
||||
#define R_TMP3 R15
|
||||
|
||||
// TEST_SRC will check if R_SRC is <= SRC_END
|
||||
#define TEST_SRC() \
|
||||
CMP R_SEND, R_SRC \
|
||||
BGT errCorrupt
|
||||
|
||||
// MOVD R_SRC, R_TMP1
|
||||
// SUB R_SBASE, R_TMP1, R_TMP1
|
||||
// CMP R_SLEN, R_TMP1
|
||||
// BGT errCorrupt
|
||||
|
||||
// The asm code generally follows the pure Go code in decode_other.go, except
|
||||
// where marked with a "!!!".
|
||||
|
||||
// func decode(dst, src []byte) int
|
||||
//
|
||||
// All local variables fit into registers. The non-zero stack size is only to
|
||||
// spill registers and push args when issuing a CALL. The register allocation:
|
||||
// - R_TMP0 scratch
|
||||
// - R_TMP1 scratch
|
||||
// - R_LEN length or x
|
||||
// - R_OFF offset
|
||||
// - R_SRC &src[s]
|
||||
// - R_DST &dst[d]
|
||||
// + R_DBASE dst_base
|
||||
// + R_DLEN dst_len
|
||||
// + R_DEND dst_base + dst_len
|
||||
// + R_SBASE src_base
|
||||
// + R_SLEN src_len
|
||||
// + R_SEND src_base + src_len
|
||||
// - R_TMP2 used by doCopy
|
||||
// - R_TMP3 used by doCopy
|
||||
//
|
||||
// The registers R_DBASE-R_SEND (marked with a "+") are set at the start of the
|
||||
// function, and after a CALL returns, and are not otherwise modified.
|
||||
//
|
||||
// The d variable is implicitly R_DST - R_DBASE, and len(dst)-d is R_DEND - R_DST.
|
||||
// The s variable is implicitly R_SRC - R_SBASE, and len(src)-s is R_SEND - R_SRC.
|
||||
TEXT ·s2Decode(SB), NOSPLIT, $56-64
|
||||
// Initialize R_SRC, R_DST and R_DBASE-R_SEND.
|
||||
MOVD dst_base+0(FP), R_DBASE
|
||||
MOVD dst_len+8(FP), R_DLEN
|
||||
MOVD R_DBASE, R_DST
|
||||
MOVD R_DBASE, R_DEND
|
||||
ADD R_DLEN, R_DEND, R_DEND
|
||||
MOVD src_base+24(FP), R_SBASE
|
||||
MOVD src_len+32(FP), R_SLEN
|
||||
MOVD R_SBASE, R_SRC
|
||||
MOVD R_SBASE, R_SEND
|
||||
ADD R_SLEN, R_SEND, R_SEND
|
||||
MOVD $0, R_OFF
|
||||
|
||||
loop:
|
||||
// for s < len(src)
|
||||
CMP R_SEND, R_SRC
|
||||
BEQ end
|
||||
|
||||
// R_LEN = uint32(src[s])
|
||||
//
|
||||
// switch src[s] & 0x03
|
||||
MOVBU (R_SRC), R_LEN
|
||||
MOVW R_LEN, R_TMP1
|
||||
ANDW $3, R_TMP1
|
||||
MOVW $1, R1
|
||||
CMPW R1, R_TMP1
|
||||
BGE tagCopy
|
||||
|
||||
// ----------------------------------------
|
||||
// The code below handles literal tags.
|
||||
|
||||
// case tagLiteral:
|
||||
// x := uint32(src[s] >> 2)
|
||||
// switch
|
||||
MOVW $60, R1
|
||||
LSRW $2, R_LEN, R_LEN
|
||||
CMPW R_LEN, R1
|
||||
BLS tagLit60Plus
|
||||
|
||||
// case x < 60:
|
||||
// s++
|
||||
ADD $1, R_SRC, R_SRC
|
||||
|
||||
doLit:
|
||||
// This is the end of the inner "switch", when we have a literal tag.
|
||||
//
|
||||
// We assume that R_LEN == x and x fits in a uint32, where x is the variable
|
||||
// used in the pure Go decode_other.go code.
|
||||
|
||||
// length = int(x) + 1
|
||||
//
|
||||
// Unlike the pure Go code, we don't need to check if length <= 0 because
|
||||
// R_LEN can hold 64 bits, so the increment cannot overflow.
|
||||
ADD $1, R_LEN, R_LEN
|
||||
|
||||
// Prepare to check if copying length bytes will run past the end of dst or
|
||||
// src.
|
||||
//
|
||||
// R_TMP0 = len(dst) - d
|
||||
// R_TMP1 = len(src) - s
|
||||
MOVD R_DEND, R_TMP0
|
||||
SUB R_DST, R_TMP0, R_TMP0
|
||||
MOVD R_SEND, R_TMP1
|
||||
SUB R_SRC, R_TMP1, R_TMP1
|
||||
|
||||
// !!! Try a faster technique for short (16 or fewer bytes) copies.
|
||||
//
|
||||
// if length > 16 || len(dst)-d < 16 || len(src)-s < 16 {
|
||||
// goto callMemmove // Fall back on calling runtime·memmove.
|
||||
// }
|
||||
//
|
||||
// The C++ snappy code calls this TryFastAppend. It also checks len(src)-s
|
||||
// against 21 instead of 16, because it cannot assume that all of its input
|
||||
// is contiguous in memory and so it needs to leave enough source bytes to
|
||||
// read the next tag without refilling buffers, but Go's Decode assumes
|
||||
// contiguousness (the src argument is a []byte).
|
||||
CMP $16, R_LEN
|
||||
BGT callMemmove
|
||||
CMP $16, R_TMP0
|
||||
BLT callMemmove
|
||||
CMP $16, R_TMP1
|
||||
BLT callMemmove
|
||||
|
||||
// !!! Implement the copy from src to dst as a 16-byte load and store.
|
||||
// (Decode's documentation says that dst and src must not overlap.)
|
||||
//
|
||||
// This always copies 16 bytes, instead of only length bytes, but that's
|
||||
// OK. If the input is a valid Snappy encoding then subsequent iterations
|
||||
// will fix up the overrun. Otherwise, Decode returns a nil []byte (and a
|
||||
// non-nil error), so the overrun will be ignored.
|
||||
//
|
||||
// Note that on arm64, it is legal and cheap to issue unaligned 8-byte or
|
||||
// 16-byte loads and stores. This technique probably wouldn't be as
|
||||
// effective on architectures that are fussier about alignment.
|
||||
LDP 0(R_SRC), (R_TMP2, R_TMP3)
|
||||
STP (R_TMP2, R_TMP3), 0(R_DST)
|
||||
|
||||
// d += length
|
||||
// s += length
|
||||
ADD R_LEN, R_DST, R_DST
|
||||
ADD R_LEN, R_SRC, R_SRC
|
||||
B loop
|
||||
|
||||
callMemmove:
|
||||
// if length > len(dst)-d || length > len(src)-s { etc }
|
||||
CMP R_TMP0, R_LEN
|
||||
BGT errCorrupt
|
||||
CMP R_TMP1, R_LEN
|
||||
BGT errCorrupt
|
||||
|
||||
// copy(dst[d:], src[s:s+length])
|
||||
//
|
||||
// This means calling runtime·memmove(&dst[d], &src[s], length), so we push
|
||||
// R_DST, R_SRC and R_LEN as arguments. Coincidentally, we also need to spill those
|
||||
// three registers to the stack, to save local variables across the CALL.
|
||||
MOVD R_DST, 8(RSP)
|
||||
MOVD R_SRC, 16(RSP)
|
||||
MOVD R_LEN, 24(RSP)
|
||||
MOVD R_DST, 32(RSP)
|
||||
MOVD R_SRC, 40(RSP)
|
||||
MOVD R_LEN, 48(RSP)
|
||||
MOVD R_OFF, 56(RSP)
|
||||
CALL runtime·memmove(SB)
|
||||
|
||||
// Restore local variables: unspill registers from the stack and
|
||||
// re-calculate R_DBASE-R_SEND.
|
||||
MOVD 32(RSP), R_DST
|
||||
MOVD 40(RSP), R_SRC
|
||||
MOVD 48(RSP), R_LEN
|
||||
MOVD 56(RSP), R_OFF
|
||||
MOVD dst_base+0(FP), R_DBASE
|
||||
MOVD dst_len+8(FP), R_DLEN
|
||||
MOVD R_DBASE, R_DEND
|
||||
ADD R_DLEN, R_DEND, R_DEND
|
||||
MOVD src_base+24(FP), R_SBASE
|
||||
MOVD src_len+32(FP), R_SLEN
|
||||
MOVD R_SBASE, R_SEND
|
||||
ADD R_SLEN, R_SEND, R_SEND
|
||||
|
||||
// d += length
|
||||
// s += length
|
||||
ADD R_LEN, R_DST, R_DST
|
||||
ADD R_LEN, R_SRC, R_SRC
|
||||
B loop
|
||||
|
||||
tagLit60Plus:
|
||||
// !!! This fragment does the
|
||||
//
|
||||
// s += x - 58; if uint(s) > uint(len(src)) { etc }
|
||||
//
|
||||
// checks. In the asm version, we code it once instead of once per switch case.
|
||||
ADD R_LEN, R_SRC, R_SRC
|
||||
SUB $58, R_SRC, R_SRC
|
||||
TEST_SRC()
|
||||
|
||||
// case x == 60:
|
||||
MOVW $61, R1
|
||||
CMPW R1, R_LEN
|
||||
BEQ tagLit61
|
||||
BGT tagLit62Plus
|
||||
|
||||
// x = uint32(src[s-1])
|
||||
MOVBU -1(R_SRC), R_LEN
|
||||
B doLit
|
||||
|
||||
tagLit61:
|
||||
// case x == 61:
|
||||
// x = uint32(src[s-2]) | uint32(src[s-1])<<8
|
||||
MOVHU -2(R_SRC), R_LEN
|
||||
B doLit
|
||||
|
||||
tagLit62Plus:
|
||||
CMPW $62, R_LEN
|
||||
BHI tagLit63
|
||||
|
||||
// case x == 62:
|
||||
// x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16
|
||||
MOVHU -3(R_SRC), R_LEN
|
||||
MOVBU -1(R_SRC), R_TMP1
|
||||
ORR R_TMP1<<16, R_LEN
|
||||
B doLit
|
||||
|
||||
tagLit63:
|
||||
// case x == 63:
|
||||
// x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24
|
||||
MOVWU -4(R_SRC), R_LEN
|
||||
B doLit
|
||||
|
||||
// The code above handles literal tags.
|
||||
// ----------------------------------------
|
||||
// The code below handles copy tags.
|
||||
|
||||
tagCopy4:
|
||||
// case tagCopy4:
|
||||
// s += 5
|
||||
ADD $5, R_SRC, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
MOVD R_SRC, R_TMP1
|
||||
SUB R_SBASE, R_TMP1, R_TMP1
|
||||
CMP R_SLEN, R_TMP1
|
||||
BGT errCorrupt
|
||||
|
||||
// length = 1 + int(src[s-5])>>2
|
||||
MOVD $1, R1
|
||||
ADD R_LEN>>2, R1, R_LEN
|
||||
|
||||
// offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24)
|
||||
MOVWU -4(R_SRC), R_OFF
|
||||
B doCopy
|
||||
|
||||
tagCopy2:
|
||||
// case tagCopy2:
|
||||
// s += 3
|
||||
ADD $3, R_SRC, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
TEST_SRC()
|
||||
|
||||
// length = 1 + int(src[s-3])>>2
|
||||
MOVD $1, R1
|
||||
ADD R_LEN>>2, R1, R_LEN
|
||||
|
||||
// offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8)
|
||||
MOVHU -2(R_SRC), R_OFF
|
||||
B doCopy
|
||||
|
||||
tagCopy:
|
||||
// We have a copy tag. We assume that:
|
||||
// - R_TMP1 == src[s] & 0x03
|
||||
// - R_LEN == src[s]
|
||||
CMP $2, R_TMP1
|
||||
BEQ tagCopy2
|
||||
BGT tagCopy4
|
||||
|
||||
// case tagCopy1:
|
||||
// s += 2
|
||||
ADD $2, R_SRC, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
TEST_SRC()
|
||||
|
||||
// offset = int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1]))
|
||||
// Calculate offset in R_TMP0 in case it is a repeat.
|
||||
MOVD R_LEN, R_TMP0
|
||||
AND $0xe0, R_TMP0
|
||||
MOVBU -1(R_SRC), R_TMP1
|
||||
ORR R_TMP0<<3, R_TMP1, R_TMP0
|
||||
|
||||
// length = 4 + int(src[s-2])>>2&0x7
|
||||
MOVD $7, R1
|
||||
AND R_LEN>>2, R1, R_LEN
|
||||
ADD $4, R_LEN, R_LEN
|
||||
|
||||
// check if repeat code with offset 0.
|
||||
CMP $0, R_TMP0
|
||||
BEQ repeatCode
|
||||
|
||||
// This is a regular copy, transfer our temporary value to R_OFF (offset)
|
||||
MOVD R_TMP0, R_OFF
|
||||
B doCopy
|
||||
|
||||
// This is a repeat code.
|
||||
repeatCode:
|
||||
// If length < 9, reuse last offset, with the length already calculated.
|
||||
CMP $9, R_LEN
|
||||
BLT doCopyRepeat
|
||||
BEQ repeatLen1
|
||||
CMP $10, R_LEN
|
||||
BEQ repeatLen2
|
||||
|
||||
repeatLen3:
|
||||
// s +=3
|
||||
ADD $3, R_SRC, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
TEST_SRC()
|
||||
|
||||
// length = uint32(src[s-3]) | (uint32(src[s-2])<<8) | (uint32(src[s-1])<<16) + 65540
|
||||
MOVBU -1(R_SRC), R_TMP0
|
||||
MOVHU -3(R_SRC), R_LEN
|
||||
ORR R_TMP0<<16, R_LEN, R_LEN
|
||||
ADD $65540, R_LEN, R_LEN
|
||||
B doCopyRepeat
|
||||
|
||||
repeatLen2:
|
||||
// s +=2
|
||||
ADD $2, R_SRC, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
TEST_SRC()
|
||||
|
||||
// length = uint32(src[s-2]) | (uint32(src[s-1])<<8) + 260
|
||||
MOVHU -2(R_SRC), R_LEN
|
||||
ADD $260, R_LEN, R_LEN
|
||||
B doCopyRepeat
|
||||
|
||||
repeatLen1:
|
||||
// s +=1
|
||||
ADD $1, R_SRC, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
TEST_SRC()
|
||||
|
||||
// length = src[s-1] + 8
|
||||
MOVBU -1(R_SRC), R_LEN
|
||||
ADD $8, R_LEN, R_LEN
|
||||
B doCopyRepeat
|
||||
|
||||
doCopy:
|
||||
// This is the end of the outer "switch", when we have a copy tag.
|
||||
//
|
||||
// We assume that:
|
||||
// - R_LEN == length && R_LEN > 0
|
||||
// - R_OFF == offset
|
||||
|
||||
// if d < offset { etc }
|
||||
MOVD R_DST, R_TMP1
|
||||
SUB R_DBASE, R_TMP1, R_TMP1
|
||||
CMP R_OFF, R_TMP1
|
||||
BLT errCorrupt
|
||||
|
||||
// Repeat values can skip the test above, since any offset > 0 will be in dst.
|
||||
doCopyRepeat:
|
||||
|
||||
// if offset <= 0 { etc }
|
||||
CMP $0, R_OFF
|
||||
BLE errCorrupt
|
||||
|
||||
// if length > len(dst)-d { etc }
|
||||
MOVD R_DEND, R_TMP1
|
||||
SUB R_DST, R_TMP1, R_TMP1
|
||||
CMP R_TMP1, R_LEN
|
||||
BGT errCorrupt
|
||||
|
||||
// forwardCopy(dst[d:d+length], dst[d-offset:]); d += length
|
||||
//
|
||||
// Set:
|
||||
// - R_TMP2 = len(dst)-d
|
||||
// - R_TMP3 = &dst[d-offset]
|
||||
MOVD R_DEND, R_TMP2
|
||||
SUB R_DST, R_TMP2, R_TMP2
|
||||
MOVD R_DST, R_TMP3
|
||||
SUB R_OFF, R_TMP3, R_TMP3
|
||||
|
||||
// !!! Try a faster technique for short (16 or fewer bytes) forward copies.
|
||||
//
|
||||
// First, try using two 8-byte load/stores, similar to the doLit technique
|
||||
// above. Even if dst[d:d+length] and dst[d-offset:] can overlap, this is
|
||||
// still OK if offset >= 8. Note that this has to be two 8-byte load/stores
|
||||
// and not one 16-byte load/store, and the first store has to be before the
|
||||
// second load, due to the overlap if offset is in the range [8, 16).
|
||||
//
|
||||
// if length > 16 || offset < 8 || len(dst)-d < 16 {
|
||||
// goto slowForwardCopy
|
||||
// }
|
||||
// copy 16 bytes
|
||||
// d += length
|
||||
CMP $16, R_LEN
|
||||
BGT slowForwardCopy
|
||||
CMP $8, R_OFF
|
||||
BLT slowForwardCopy
|
||||
CMP $16, R_TMP2
|
||||
BLT slowForwardCopy
|
||||
MOVD 0(R_TMP3), R_TMP0
|
||||
MOVD R_TMP0, 0(R_DST)
|
||||
MOVD 8(R_TMP3), R_TMP1
|
||||
MOVD R_TMP1, 8(R_DST)
|
||||
ADD R_LEN, R_DST, R_DST
|
||||
B loop
|
||||
|
||||
slowForwardCopy:
|
||||
// !!! If the forward copy is longer than 16 bytes, or if offset < 8, we
|
||||
// can still try 8-byte load stores, provided we can overrun up to 10 extra
|
||||
// bytes. As above, the overrun will be fixed up by subsequent iterations
|
||||
// of the outermost loop.
|
||||
//
|
||||
// The C++ snappy code calls this technique IncrementalCopyFastPath. Its
|
||||
// commentary says:
|
||||
//
|
||||
// ----
|
||||
//
|
||||
// The main part of this loop is a simple copy of eight bytes at a time
|
||||
// until we've copied (at least) the requested amount of bytes. However,
|
||||
// if d and d-offset are less than eight bytes apart (indicating a
|
||||
// repeating pattern of length < 8), we first need to expand the pattern in
|
||||
// order to get the correct results. For instance, if the buffer looks like
|
||||
// this, with the eight-byte <d-offset> and <d> patterns marked as
|
||||
// intervals:
|
||||
//
|
||||
// abxxxxxxxxxxxx
|
||||
// [------] d-offset
|
||||
// [------] d
|
||||
//
|
||||
// a single eight-byte copy from <d-offset> to <d> will repeat the pattern
|
||||
// once, after which we can move <d> two bytes without moving <d-offset>:
|
||||
//
|
||||
// ababxxxxxxxxxx
|
||||
// [------] d-offset
|
||||
// [------] d
|
||||
//
|
||||
// and repeat the exercise until the two no longer overlap.
|
||||
//
|
||||
// This allows us to do very well in the special case of one single byte
|
||||
// repeated many times, without taking a big hit for more general cases.
|
||||
//
|
||||
// The worst case of extra writing past the end of the match occurs when
|
||||
// offset == 1 and length == 1; the last copy will read from byte positions
|
||||
// [0..7] and write to [4..11], whereas it was only supposed to write to
|
||||
// position 1. Thus, ten excess bytes.
|
||||
//
|
||||
// ----
|
||||
//
|
||||
// That "10 byte overrun" worst case is confirmed by Go's
|
||||
// TestSlowForwardCopyOverrun, which also tests the fixUpSlowForwardCopy
|
||||
// and finishSlowForwardCopy algorithm.
|
||||
//
|
||||
// if length > len(dst)-d-10 {
|
||||
// goto verySlowForwardCopy
|
||||
// }
|
||||
SUB $10, R_TMP2, R_TMP2
|
||||
CMP R_TMP2, R_LEN
|
||||
BGT verySlowForwardCopy
|
||||
|
||||
// We want to keep the offset, so we use R_TMP2 from here.
|
||||
MOVD R_OFF, R_TMP2
|
||||
|
||||
makeOffsetAtLeast8:
|
||||
// !!! As above, expand the pattern so that offset >= 8 and we can use
|
||||
// 8-byte load/stores.
|
||||
//
|
||||
// for offset < 8 {
|
||||
// copy 8 bytes from dst[d-offset:] to dst[d:]
|
||||
// length -= offset
|
||||
// d += offset
|
||||
// offset += offset
|
||||
// // The two previous lines together means that d-offset, and therefore
|
||||
// // R_TMP3, is unchanged.
|
||||
// }
|
||||
CMP $8, R_TMP2
|
||||
BGE fixUpSlowForwardCopy
|
||||
MOVD (R_TMP3), R_TMP1
|
||||
MOVD R_TMP1, (R_DST)
|
||||
SUB R_TMP2, R_LEN, R_LEN
|
||||
ADD R_TMP2, R_DST, R_DST
|
||||
ADD R_TMP2, R_TMP2, R_TMP2
|
||||
B makeOffsetAtLeast8
|
||||
|
||||
fixUpSlowForwardCopy:
|
||||
// !!! Add length (which might be negative now) to d (implied by R_DST being
|
||||
// &dst[d]) so that d ends up at the right place when we jump back to the
|
||||
// top of the loop. Before we do that, though, we save R_DST to R_TMP0 so that, if
|
||||
// length is positive, copying the remaining length bytes will write to the
|
||||
// right place.
|
||||
MOVD R_DST, R_TMP0
|
||||
ADD R_LEN, R_DST, R_DST
|
||||
|
||||
finishSlowForwardCopy:
|
||||
// !!! Repeat 8-byte load/stores until length <= 0. Ending with a negative
|
||||
// length means that we overrun, but as above, that will be fixed up by
|
||||
// subsequent iterations of the outermost loop.
|
||||
MOVD $0, R1
|
||||
CMP R1, R_LEN
|
||||
BLE loop
|
||||
MOVD (R_TMP3), R_TMP1
|
||||
MOVD R_TMP1, (R_TMP0)
|
||||
ADD $8, R_TMP3, R_TMP3
|
||||
ADD $8, R_TMP0, R_TMP0
|
||||
SUB $8, R_LEN, R_LEN
|
||||
B finishSlowForwardCopy
|
||||
|
||||
verySlowForwardCopy:
|
||||
// verySlowForwardCopy is a simple implementation of forward copy. In C
|
||||
// parlance, this is a do/while loop instead of a while loop, since we know
|
||||
// that length > 0. In Go syntax:
|
||||
//
|
||||
// for {
|
||||
// dst[d] = dst[d - offset]
|
||||
// d++
|
||||
// length--
|
||||
// if length == 0 {
|
||||
// break
|
||||
// }
|
||||
// }
|
||||
MOVB (R_TMP3), R_TMP1
|
||||
MOVB R_TMP1, (R_DST)
|
||||
ADD $1, R_TMP3, R_TMP3
|
||||
ADD $1, R_DST, R_DST
|
||||
SUB $1, R_LEN, R_LEN
|
||||
CBNZ R_LEN, verySlowForwardCopy
|
||||
B loop
|
||||
|
||||
// The code above handles copy tags.
|
||||
// ----------------------------------------
|
||||
|
||||
end:
|
||||
// This is the end of the "for s < len(src)".
|
||||
//
|
||||
// if d != len(dst) { etc }
|
||||
CMP R_DEND, R_DST
|
||||
BNE errCorrupt
|
||||
|
||||
// return 0
|
||||
MOVD $0, ret+48(FP)
|
||||
RET
|
||||
|
||||
errCorrupt:
|
||||
// return decodeErrCodeCorrupt
|
||||
MOVD $1, R_TMP0
|
||||
MOVD R_TMP0, ret+48(FP)
|
||||
RET
|
||||
17
vendor/github.com/klauspost/compress/s2/decode_asm.go
сгенерированный
поставляемый
Обычный файл
17
vendor/github.com/klauspost/compress/s2/decode_asm.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,17 @@
|
||||
// Copyright 2016 The Snappy-Go Authors. All rights reserved.
|
||||
// Copyright (c) 2019 Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
//go:build (amd64 || arm64) && !appengine && gc && !noasm
|
||||
// +build amd64 arm64
|
||||
// +build !appengine
|
||||
// +build gc
|
||||
// +build !noasm
|
||||
|
||||
package s2
|
||||
|
||||
// decode has the same semantics as in decode_other.go.
|
||||
//
|
||||
//go:noescape
|
||||
func s2Decode(dst, src []byte) int
|
||||
267
vendor/github.com/klauspost/compress/s2/decode_other.go
сгенерированный
поставляемый
Обычный файл
267
vendor/github.com/klauspost/compress/s2/decode_other.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,267 @@
|
||||
// Copyright 2016 The Snappy-Go Authors. All rights reserved.
|
||||
// Copyright (c) 2019 Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
//go:build (!amd64 && !arm64) || appengine || !gc || noasm
|
||||
// +build !amd64,!arm64 appengine !gc noasm
|
||||
|
||||
package s2
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// decode writes the decoding of src to dst. It assumes that the varint-encoded
|
||||
// length of the decompressed bytes has already been read, and that len(dst)
|
||||
// equals that length.
|
||||
//
|
||||
// It returns 0 on success or a decodeErrCodeXxx error code on failure.
|
||||
func s2Decode(dst, src []byte) int {
|
||||
const debug = false
|
||||
if debug {
|
||||
fmt.Println("Starting decode, dst len:", len(dst))
|
||||
}
|
||||
var d, s, length int
|
||||
offset := 0
|
||||
|
||||
// As long as we can read at least 5 bytes...
|
||||
for s < len(src)-5 {
|
||||
switch src[s] & 0x03 {
|
||||
case tagLiteral:
|
||||
x := uint32(src[s] >> 2)
|
||||
switch {
|
||||
case x < 60:
|
||||
s++
|
||||
case x == 60:
|
||||
s += 2
|
||||
x = uint32(src[s-1])
|
||||
case x == 61:
|
||||
s += 3
|
||||
x = uint32(src[s-2]) | uint32(src[s-1])<<8
|
||||
case x == 62:
|
||||
s += 4
|
||||
x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16
|
||||
case x == 63:
|
||||
s += 5
|
||||
x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24
|
||||
}
|
||||
length = int(x) + 1
|
||||
if length > len(dst)-d || length > len(src)-s || (strconv.IntSize == 32 && length <= 0) {
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
if debug {
|
||||
fmt.Println("literals, length:", length, "d-after:", d+length)
|
||||
}
|
||||
|
||||
copy(dst[d:], src[s:s+length])
|
||||
d += length
|
||||
s += length
|
||||
continue
|
||||
|
||||
case tagCopy1:
|
||||
s += 2
|
||||
length = int(src[s-2]) >> 2 & 0x7
|
||||
toffset := int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1]))
|
||||
if toffset == 0 {
|
||||
if debug {
|
||||
fmt.Print("(repeat) ")
|
||||
}
|
||||
// keep last offset
|
||||
switch length {
|
||||
case 5:
|
||||
s += 1
|
||||
length = int(uint32(src[s-1])) + 4
|
||||
case 6:
|
||||
s += 2
|
||||
length = int(uint32(src[s-2])|(uint32(src[s-1])<<8)) + (1 << 8)
|
||||
case 7:
|
||||
s += 3
|
||||
length = int(uint32(src[s-3])|(uint32(src[s-2])<<8)|(uint32(src[s-1])<<16)) + (1 << 16)
|
||||
default: // 0-> 4
|
||||
}
|
||||
} else {
|
||||
offset = toffset
|
||||
}
|
||||
length += 4
|
||||
case tagCopy2:
|
||||
s += 3
|
||||
length = 1 + int(src[s-3])>>2
|
||||
offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8)
|
||||
|
||||
case tagCopy4:
|
||||
s += 5
|
||||
length = 1 + int(src[s-5])>>2
|
||||
offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24)
|
||||
}
|
||||
|
||||
if offset <= 0 || d < offset || length > len(dst)-d {
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
|
||||
if debug {
|
||||
fmt.Println("copy, length:", length, "offset:", offset, "d-after:", d+length)
|
||||
}
|
||||
|
||||
// Copy from an earlier sub-slice of dst to a later sub-slice.
|
||||
// If no overlap, use the built-in copy:
|
||||
if offset > length {
|
||||
copy(dst[d:d+length], dst[d-offset:])
|
||||
d += length
|
||||
continue
|
||||
}
|
||||
|
||||
// Unlike the built-in copy function, this byte-by-byte copy always runs
|
||||
// forwards, even if the slices overlap. Conceptually, this is:
|
||||
//
|
||||
// d += forwardCopy(dst[d:d+length], dst[d-offset:])
|
||||
//
|
||||
// We align the slices into a and b and show the compiler they are the same size.
|
||||
// This allows the loop to run without bounds checks.
|
||||
a := dst[d : d+length]
|
||||
b := dst[d-offset:]
|
||||
b = b[:len(a)]
|
||||
for i := range a {
|
||||
a[i] = b[i]
|
||||
}
|
||||
d += length
|
||||
}
|
||||
|
||||
// Remaining with extra checks...
|
||||
for s < len(src) {
|
||||
switch src[s] & 0x03 {
|
||||
case tagLiteral:
|
||||
x := uint32(src[s] >> 2)
|
||||
switch {
|
||||
case x < 60:
|
||||
s++
|
||||
case x == 60:
|
||||
s += 2
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
x = uint32(src[s-1])
|
||||
case x == 61:
|
||||
s += 3
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
x = uint32(src[s-2]) | uint32(src[s-1])<<8
|
||||
case x == 62:
|
||||
s += 4
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16
|
||||
case x == 63:
|
||||
s += 5
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24
|
||||
}
|
||||
length = int(x) + 1
|
||||
if length > len(dst)-d || length > len(src)-s || (strconv.IntSize == 32 && length <= 0) {
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
if debug {
|
||||
fmt.Println("literals, length:", length, "d-after:", d+length)
|
||||
}
|
||||
|
||||
copy(dst[d:], src[s:s+length])
|
||||
d += length
|
||||
s += length
|
||||
continue
|
||||
|
||||
case tagCopy1:
|
||||
s += 2
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = int(src[s-2]) >> 2 & 0x7
|
||||
toffset := int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1]))
|
||||
if toffset == 0 {
|
||||
if debug {
|
||||
fmt.Print("(repeat) ")
|
||||
}
|
||||
// keep last offset
|
||||
switch length {
|
||||
case 5:
|
||||
s += 1
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = int(uint32(src[s-1])) + 4
|
||||
case 6:
|
||||
s += 2
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = int(uint32(src[s-2])|(uint32(src[s-1])<<8)) + (1 << 8)
|
||||
case 7:
|
||||
s += 3
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = int(uint32(src[s-3])|(uint32(src[s-2])<<8)|(uint32(src[s-1])<<16)) + (1 << 16)
|
||||
default: // 0-> 4
|
||||
}
|
||||
} else {
|
||||
offset = toffset
|
||||
}
|
||||
length += 4
|
||||
case tagCopy2:
|
||||
s += 3
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = 1 + int(src[s-3])>>2
|
||||
offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8)
|
||||
|
||||
case tagCopy4:
|
||||
s += 5
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = 1 + int(src[s-5])>>2
|
||||
offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24)
|
||||
}
|
||||
|
||||
if offset <= 0 || d < offset || length > len(dst)-d {
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
|
||||
if debug {
|
||||
fmt.Println("copy, length:", length, "offset:", offset, "d-after:", d+length)
|
||||
}
|
||||
|
||||
// Copy from an earlier sub-slice of dst to a later sub-slice.
|
||||
// If no overlap, use the built-in copy:
|
||||
if offset > length {
|
||||
copy(dst[d:d+length], dst[d-offset:])
|
||||
d += length
|
||||
continue
|
||||
}
|
||||
|
||||
// Unlike the built-in copy function, this byte-by-byte copy always runs
|
||||
// forwards, even if the slices overlap. Conceptually, this is:
|
||||
//
|
||||
// d += forwardCopy(dst[d:d+length], dst[d-offset:])
|
||||
//
|
||||
// We align the slices into a and b and show the compiler they are the same size.
|
||||
// This allows the loop to run without bounds checks.
|
||||
a := dst[d : d+length]
|
||||
b := dst[d-offset:]
|
||||
b = b[:len(a)]
|
||||
for i := range a {
|
||||
a[i] = b[i]
|
||||
}
|
||||
d += length
|
||||
}
|
||||
|
||||
if d != len(dst) {
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
return 0
|
||||
}
|
||||
1172
vendor/github.com/klauspost/compress/s2/encode.go
сгенерированный
поставляемый
Обычный файл
1172
vendor/github.com/klauspost/compress/s2/encode.go
сгенерированный
поставляемый
Обычный файл
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
456
vendor/github.com/klauspost/compress/s2/encode_all.go
сгенерированный
поставляемый
Обычный файл
456
vendor/github.com/klauspost/compress/s2/encode_all.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,456 @@
|
||||
// Copyright 2016 The Snappy-Go Authors. All rights reserved.
|
||||
// Copyright (c) 2019 Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package s2
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"math/bits"
|
||||
)
|
||||
|
||||
func load32(b []byte, i int) uint32 {
|
||||
return binary.LittleEndian.Uint32(b[i:])
|
||||
}
|
||||
|
||||
func load64(b []byte, i int) uint64 {
|
||||
return binary.LittleEndian.Uint64(b[i:])
|
||||
}
|
||||
|
||||
// hash6 returns the hash of the lowest 6 bytes of u to fit in a hash table with h bits.
|
||||
// Preferably h should be a constant and should always be <64.
|
||||
func hash6(u uint64, h uint8) uint32 {
|
||||
const prime6bytes = 227718039650203
|
||||
return uint32(((u << (64 - 48)) * prime6bytes) >> ((64 - h) & 63))
|
||||
}
|
||||
|
||||
func encodeGo(dst, src []byte) []byte {
|
||||
if n := MaxEncodedLen(len(src)); n < 0 {
|
||||
panic(ErrTooLarge)
|
||||
} else if len(dst) < n {
|
||||
dst = make([]byte, n)
|
||||
}
|
||||
|
||||
// The block starts with the varint-encoded length of the decompressed bytes.
|
||||
d := binary.PutUvarint(dst, uint64(len(src)))
|
||||
|
||||
if len(src) == 0 {
|
||||
return dst[:d]
|
||||
}
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
d += emitLiteral(dst[d:], src)
|
||||
return dst[:d]
|
||||
}
|
||||
n := encodeBlockGo(dst[d:], src)
|
||||
if n > 0 {
|
||||
d += n
|
||||
return dst[:d]
|
||||
}
|
||||
// Not compressible
|
||||
d += emitLiteral(dst[d:], src)
|
||||
return dst[:d]
|
||||
}
|
||||
|
||||
// encodeBlockGo encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlockGo(dst, src []byte) (d int) {
|
||||
// Initialize the hash table.
|
||||
const (
|
||||
tableBits = 14
|
||||
maxTableSize = 1 << tableBits
|
||||
|
||||
debug = false
|
||||
)
|
||||
|
||||
var table [maxTableSize]uint32
|
||||
|
||||
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
||||
// lets us use a fast path for emitLiteral in the main loop, while we are
|
||||
// looking for copies.
|
||||
sLimit := len(src) - inputMargin
|
||||
|
||||
// Bail if we can't compress to at least this.
|
||||
dstLimit := len(src) - len(src)>>5 - 5
|
||||
|
||||
// nextEmit is where in src the next emitLiteral should start from.
|
||||
nextEmit := 0
|
||||
|
||||
// The encoded form must start with a literal, as there are no previous
|
||||
// bytes to copy, so we start looking for hash matches at s == 1.
|
||||
s := 1
|
||||
cv := load64(src, s)
|
||||
|
||||
// We search for a repeat at -1, but don't output repeats when nextEmit == 0
|
||||
repeat := 1
|
||||
|
||||
for {
|
||||
candidate := 0
|
||||
for {
|
||||
// Next src position to check
|
||||
nextS := s + (s-nextEmit)>>6 + 4
|
||||
if nextS > sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
hash0 := hash6(cv, tableBits)
|
||||
hash1 := hash6(cv>>8, tableBits)
|
||||
candidate = int(table[hash0])
|
||||
candidate2 := int(table[hash1])
|
||||
table[hash0] = uint32(s)
|
||||
table[hash1] = uint32(s + 1)
|
||||
hash2 := hash6(cv>>16, tableBits)
|
||||
|
||||
// Check repeat at offset checkRep.
|
||||
const checkRep = 1
|
||||
if uint32(cv>>(checkRep*8)) == load32(src, s-repeat+checkRep) {
|
||||
base := s + checkRep
|
||||
// Extend back
|
||||
for i := base - repeat; base > nextEmit && i > 0 && src[i-1] == src[base-1]; {
|
||||
i--
|
||||
base--
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:base])
|
||||
|
||||
// Extend forward
|
||||
candidate := s - repeat + 4 + checkRep
|
||||
s += 4 + checkRep
|
||||
for s <= sLimit {
|
||||
if diff := load64(src, s) ^ load64(src, candidate); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
if debug {
|
||||
// Validate match.
|
||||
if s <= candidate {
|
||||
panic("s <= candidate")
|
||||
}
|
||||
a := src[base:s]
|
||||
b := src[base-repeat : base-repeat+(s-base)]
|
||||
if !bytes.Equal(a, b) {
|
||||
panic("mismatch")
|
||||
}
|
||||
}
|
||||
if nextEmit > 0 {
|
||||
// same as `add := emitCopy(dst[d:], repeat, s-base)` but skips storing offset.
|
||||
d += emitRepeat(dst[d:], repeat, s-base)
|
||||
} else {
|
||||
// First match, cannot be repeat.
|
||||
d += emitCopy(dst[d:], repeat, s-base)
|
||||
}
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
cv = load64(src, s)
|
||||
continue
|
||||
}
|
||||
|
||||
if uint32(cv) == load32(src, candidate) {
|
||||
break
|
||||
}
|
||||
candidate = int(table[hash2])
|
||||
if uint32(cv>>8) == load32(src, candidate2) {
|
||||
table[hash2] = uint32(s + 2)
|
||||
candidate = candidate2
|
||||
s++
|
||||
break
|
||||
}
|
||||
table[hash2] = uint32(s + 2)
|
||||
if uint32(cv>>16) == load32(src, candidate) {
|
||||
s += 2
|
||||
break
|
||||
}
|
||||
|
||||
cv = load64(src, nextS)
|
||||
s = nextS
|
||||
}
|
||||
|
||||
// Extend backwards.
|
||||
// The top bytes will be rechecked to get the full match.
|
||||
for candidate > 0 && s > nextEmit && src[candidate-1] == src[s-1] {
|
||||
candidate--
|
||||
s--
|
||||
}
|
||||
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+(s-nextEmit) > dstLimit {
|
||||
return 0
|
||||
}
|
||||
|
||||
// A 4-byte match has been found. We'll later see if more than 4 bytes
|
||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
|
||||
// them as literal bytes.
|
||||
|
||||
d += emitLiteral(dst[d:], src[nextEmit:s])
|
||||
|
||||
// Call emitCopy, and then see if another emitCopy could be our next
|
||||
// move. Repeat until we find no match for the input immediately after
|
||||
// what was consumed by the last emitCopy call.
|
||||
//
|
||||
// If we exit this loop normally then we need to call emitLiteral next,
|
||||
// though we don't yet know how big the literal will be. We handle that
|
||||
// by proceeding to the next iteration of the main loop. We also can
|
||||
// exit this loop via goto if we get close to exhausting the input.
|
||||
for {
|
||||
// Invariant: we have a 4-byte match at s, and no need to emit any
|
||||
// literal bytes prior to s.
|
||||
base := s
|
||||
repeat = base - candidate
|
||||
|
||||
// Extend the 4-byte match as long as possible.
|
||||
s += 4
|
||||
candidate += 4
|
||||
for s <= len(src)-8 {
|
||||
if diff := load64(src, s) ^ load64(src, candidate); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
|
||||
d += emitCopy(dst[d:], repeat, s-base)
|
||||
if debug {
|
||||
// Validate match.
|
||||
if s <= candidate {
|
||||
panic("s <= candidate")
|
||||
}
|
||||
a := src[base:s]
|
||||
b := src[base-repeat : base-repeat+(s-base)]
|
||||
if !bytes.Equal(a, b) {
|
||||
panic("mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
if d > dstLimit {
|
||||
// Do we have space for more, if not bail.
|
||||
return 0
|
||||
}
|
||||
// Check for an immediate match, otherwise start search at s+1
|
||||
x := load64(src, s-2)
|
||||
m2Hash := hash6(x, tableBits)
|
||||
currHash := hash6(x>>16, tableBits)
|
||||
candidate = int(table[currHash])
|
||||
table[m2Hash] = uint32(s - 2)
|
||||
table[currHash] = uint32(s)
|
||||
if debug && s == candidate {
|
||||
panic("s == candidate")
|
||||
}
|
||||
if uint32(x>>16) != load32(src, candidate) {
|
||||
cv = load64(src, s+1)
|
||||
s++
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
emitRemainder:
|
||||
if nextEmit < len(src) {
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+len(src)-nextEmit > dstLimit {
|
||||
return 0
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:])
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func encodeBlockSnappyGo(dst, src []byte) (d int) {
|
||||
// Initialize the hash table.
|
||||
const (
|
||||
tableBits = 14
|
||||
maxTableSize = 1 << tableBits
|
||||
)
|
||||
|
||||
var table [maxTableSize]uint32
|
||||
|
||||
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
||||
// lets us use a fast path for emitLiteral in the main loop, while we are
|
||||
// looking for copies.
|
||||
sLimit := len(src) - inputMargin
|
||||
|
||||
// Bail if we can't compress to at least this.
|
||||
dstLimit := len(src) - len(src)>>5 - 5
|
||||
|
||||
// nextEmit is where in src the next emitLiteral should start from.
|
||||
nextEmit := 0
|
||||
|
||||
// The encoded form must start with a literal, as there are no previous
|
||||
// bytes to copy, so we start looking for hash matches at s == 1.
|
||||
s := 1
|
||||
cv := load64(src, s)
|
||||
|
||||
// We search for a repeat at -1, but don't output repeats when nextEmit == 0
|
||||
repeat := 1
|
||||
|
||||
for {
|
||||
candidate := 0
|
||||
for {
|
||||
// Next src position to check
|
||||
nextS := s + (s-nextEmit)>>6 + 4
|
||||
if nextS > sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
hash0 := hash6(cv, tableBits)
|
||||
hash1 := hash6(cv>>8, tableBits)
|
||||
candidate = int(table[hash0])
|
||||
candidate2 := int(table[hash1])
|
||||
table[hash0] = uint32(s)
|
||||
table[hash1] = uint32(s + 1)
|
||||
hash2 := hash6(cv>>16, tableBits)
|
||||
|
||||
// Check repeat at offset checkRep.
|
||||
const checkRep = 1
|
||||
if uint32(cv>>(checkRep*8)) == load32(src, s-repeat+checkRep) {
|
||||
base := s + checkRep
|
||||
// Extend back
|
||||
for i := base - repeat; base > nextEmit && i > 0 && src[i-1] == src[base-1]; {
|
||||
i--
|
||||
base--
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:base])
|
||||
|
||||
// Extend forward
|
||||
candidate := s - repeat + 4 + checkRep
|
||||
s += 4 + checkRep
|
||||
for s <= sLimit {
|
||||
if diff := load64(src, s) ^ load64(src, candidate); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
|
||||
d += emitCopyNoRepeat(dst[d:], repeat, s-base)
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
cv = load64(src, s)
|
||||
continue
|
||||
}
|
||||
|
||||
if uint32(cv) == load32(src, candidate) {
|
||||
break
|
||||
}
|
||||
candidate = int(table[hash2])
|
||||
if uint32(cv>>8) == load32(src, candidate2) {
|
||||
table[hash2] = uint32(s + 2)
|
||||
candidate = candidate2
|
||||
s++
|
||||
break
|
||||
}
|
||||
table[hash2] = uint32(s + 2)
|
||||
if uint32(cv>>16) == load32(src, candidate) {
|
||||
s += 2
|
||||
break
|
||||
}
|
||||
|
||||
cv = load64(src, nextS)
|
||||
s = nextS
|
||||
}
|
||||
|
||||
// Extend backwards
|
||||
for candidate > 0 && s > nextEmit && src[candidate-1] == src[s-1] {
|
||||
candidate--
|
||||
s--
|
||||
}
|
||||
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+(s-nextEmit) > dstLimit {
|
||||
return 0
|
||||
}
|
||||
|
||||
// A 4-byte match has been found. We'll later see if more than 4 bytes
|
||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
|
||||
// them as literal bytes.
|
||||
|
||||
d += emitLiteral(dst[d:], src[nextEmit:s])
|
||||
|
||||
// Call emitCopy, and then see if another emitCopy could be our next
|
||||
// move. Repeat until we find no match for the input immediately after
|
||||
// what was consumed by the last emitCopy call.
|
||||
//
|
||||
// If we exit this loop normally then we need to call emitLiteral next,
|
||||
// though we don't yet know how big the literal will be. We handle that
|
||||
// by proceeding to the next iteration of the main loop. We also can
|
||||
// exit this loop via goto if we get close to exhausting the input.
|
||||
for {
|
||||
// Invariant: we have a 4-byte match at s, and no need to emit any
|
||||
// literal bytes prior to s.
|
||||
base := s
|
||||
repeat = base - candidate
|
||||
|
||||
// Extend the 4-byte match as long as possible.
|
||||
s += 4
|
||||
candidate += 4
|
||||
for s <= len(src)-8 {
|
||||
if diff := load64(src, s) ^ load64(src, candidate); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
|
||||
d += emitCopyNoRepeat(dst[d:], repeat, s-base)
|
||||
if false {
|
||||
// Validate match.
|
||||
a := src[base:s]
|
||||
b := src[base-repeat : base-repeat+(s-base)]
|
||||
if !bytes.Equal(a, b) {
|
||||
panic("mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
if d > dstLimit {
|
||||
// Do we have space for more, if not bail.
|
||||
return 0
|
||||
}
|
||||
// Check for an immediate match, otherwise start search at s+1
|
||||
x := load64(src, s-2)
|
||||
m2Hash := hash6(x, tableBits)
|
||||
currHash := hash6(x>>16, tableBits)
|
||||
candidate = int(table[currHash])
|
||||
table[m2Hash] = uint32(s - 2)
|
||||
table[currHash] = uint32(s)
|
||||
if uint32(x>>16) != load32(src, candidate) {
|
||||
cv = load64(src, s+1)
|
||||
s++
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
emitRemainder:
|
||||
if nextEmit < len(src) {
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+len(src)-nextEmit > dstLimit {
|
||||
return 0
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:])
|
||||
}
|
||||
return d
|
||||
}
|
||||
142
vendor/github.com/klauspost/compress/s2/encode_amd64.go
сгенерированный
поставляемый
Обычный файл
142
vendor/github.com/klauspost/compress/s2/encode_amd64.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,142 @@
|
||||
//go:build !appengine && !noasm && gc
|
||||
// +build !appengine,!noasm,gc
|
||||
|
||||
package s2
|
||||
|
||||
// encodeBlock encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlock(dst, src []byte) (d int) {
|
||||
const (
|
||||
// Use 12 bit table when less than...
|
||||
limit12B = 16 << 10
|
||||
// Use 10 bit table when less than...
|
||||
limit10B = 4 << 10
|
||||
// Use 8 bit table when less than...
|
||||
limit8B = 512
|
||||
)
|
||||
|
||||
if len(src) >= 4<<20 {
|
||||
return encodeBlockAsm(dst, src)
|
||||
}
|
||||
if len(src) >= limit12B {
|
||||
return encodeBlockAsm4MB(dst, src)
|
||||
}
|
||||
if len(src) >= limit10B {
|
||||
return encodeBlockAsm12B(dst, src)
|
||||
}
|
||||
if len(src) >= limit8B {
|
||||
return encodeBlockAsm10B(dst, src)
|
||||
}
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
return 0
|
||||
}
|
||||
return encodeBlockAsm8B(dst, src)
|
||||
}
|
||||
|
||||
// encodeBlockBetter encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlockBetter(dst, src []byte) (d int) {
|
||||
const (
|
||||
// Use 12 bit table when less than...
|
||||
limit12B = 16 << 10
|
||||
// Use 10 bit table when less than...
|
||||
limit10B = 4 << 10
|
||||
// Use 8 bit table when less than...
|
||||
limit8B = 512
|
||||
)
|
||||
|
||||
if len(src) > 4<<20 {
|
||||
return encodeBetterBlockAsm(dst, src)
|
||||
}
|
||||
if len(src) >= limit12B {
|
||||
return encodeBetterBlockAsm4MB(dst, src)
|
||||
}
|
||||
if len(src) >= limit10B {
|
||||
return encodeBetterBlockAsm12B(dst, src)
|
||||
}
|
||||
if len(src) >= limit8B {
|
||||
return encodeBetterBlockAsm10B(dst, src)
|
||||
}
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
return 0
|
||||
}
|
||||
return encodeBetterBlockAsm8B(dst, src)
|
||||
}
|
||||
|
||||
// encodeBlockSnappy encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlockSnappy(dst, src []byte) (d int) {
|
||||
const (
|
||||
// Use 12 bit table when less than...
|
||||
limit12B = 16 << 10
|
||||
// Use 10 bit table when less than...
|
||||
limit10B = 4 << 10
|
||||
// Use 8 bit table when less than...
|
||||
limit8B = 512
|
||||
)
|
||||
if len(src) >= 64<<10 {
|
||||
return encodeSnappyBlockAsm(dst, src)
|
||||
}
|
||||
if len(src) >= limit12B {
|
||||
return encodeSnappyBlockAsm64K(dst, src)
|
||||
}
|
||||
if len(src) >= limit10B {
|
||||
return encodeSnappyBlockAsm12B(dst, src)
|
||||
}
|
||||
if len(src) >= limit8B {
|
||||
return encodeSnappyBlockAsm10B(dst, src)
|
||||
}
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
return 0
|
||||
}
|
||||
return encodeSnappyBlockAsm8B(dst, src)
|
||||
}
|
||||
|
||||
// encodeBlockSnappy encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlockBetterSnappy(dst, src []byte) (d int) {
|
||||
const (
|
||||
// Use 12 bit table when less than...
|
||||
limit12B = 16 << 10
|
||||
// Use 10 bit table when less than...
|
||||
limit10B = 4 << 10
|
||||
// Use 8 bit table when less than...
|
||||
limit8B = 512
|
||||
)
|
||||
if len(src) >= 64<<10 {
|
||||
return encodeSnappyBetterBlockAsm(dst, src)
|
||||
}
|
||||
if len(src) >= limit12B {
|
||||
return encodeSnappyBetterBlockAsm64K(dst, src)
|
||||
}
|
||||
if len(src) >= limit10B {
|
||||
return encodeSnappyBetterBlockAsm12B(dst, src)
|
||||
}
|
||||
if len(src) >= limit8B {
|
||||
return encodeSnappyBetterBlockAsm10B(dst, src)
|
||||
}
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
return 0
|
||||
}
|
||||
return encodeSnappyBetterBlockAsm8B(dst, src)
|
||||
}
|
||||
604
vendor/github.com/klauspost/compress/s2/encode_best.go
сгенерированный
поставляемый
Обычный файл
604
vendor/github.com/klauspost/compress/s2/encode_best.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,604 @@
|
||||
// Copyright 2016 The Snappy-Go Authors. All rights reserved.
|
||||
// Copyright (c) 2019 Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package s2
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/bits"
|
||||
)
|
||||
|
||||
// encodeBlockBest encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlockBest(dst, src []byte) (d int) {
|
||||
// Initialize the hash tables.
|
||||
const (
|
||||
// Long hash matches.
|
||||
lTableBits = 19
|
||||
maxLTableSize = 1 << lTableBits
|
||||
|
||||
// Short hash matches.
|
||||
sTableBits = 16
|
||||
maxSTableSize = 1 << sTableBits
|
||||
|
||||
inputMargin = 8 + 2
|
||||
)
|
||||
|
||||
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
||||
// lets us use a fast path for emitLiteral in the main loop, while we are
|
||||
// looking for copies.
|
||||
sLimit := len(src) - inputMargin
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
return 0
|
||||
}
|
||||
|
||||
var lTable [maxLTableSize]uint64
|
||||
var sTable [maxSTableSize]uint64
|
||||
|
||||
// Bail if we can't compress to at least this.
|
||||
dstLimit := len(src) - 5
|
||||
|
||||
// nextEmit is where in src the next emitLiteral should start from.
|
||||
nextEmit := 0
|
||||
|
||||
// The encoded form must start with a literal, as there are no previous
|
||||
// bytes to copy, so we start looking for hash matches at s == 1.
|
||||
s := 1
|
||||
cv := load64(src, s)
|
||||
|
||||
// We search for a repeat at -1, but don't output repeats when nextEmit == 0
|
||||
repeat := 1
|
||||
const lowbitMask = 0xffffffff
|
||||
getCur := func(x uint64) int {
|
||||
return int(x & lowbitMask)
|
||||
}
|
||||
getPrev := func(x uint64) int {
|
||||
return int(x >> 32)
|
||||
}
|
||||
const maxSkip = 64
|
||||
|
||||
for {
|
||||
type match struct {
|
||||
offset int
|
||||
s int
|
||||
length int
|
||||
score int
|
||||
rep bool
|
||||
}
|
||||
var best match
|
||||
for {
|
||||
// Next src position to check
|
||||
nextS := (s-nextEmit)>>8 + 1
|
||||
if nextS > maxSkip {
|
||||
nextS = s + maxSkip
|
||||
} else {
|
||||
nextS += s
|
||||
}
|
||||
if nextS > sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
hashL := hash8(cv, lTableBits)
|
||||
hashS := hash4(cv, sTableBits)
|
||||
candidateL := lTable[hashL]
|
||||
candidateS := sTable[hashS]
|
||||
|
||||
score := func(m match) int {
|
||||
// Matches that are longer forward are penalized since we must emit it as a literal.
|
||||
score := m.length - m.s
|
||||
if nextEmit == m.s {
|
||||
// If we do not have to emit literals, we save 1 byte
|
||||
score++
|
||||
}
|
||||
offset := m.s - m.offset
|
||||
if m.rep {
|
||||
return score - emitRepeatSize(offset, m.length)
|
||||
}
|
||||
return score - emitCopySize(offset, m.length)
|
||||
}
|
||||
|
||||
matchAt := func(offset, s int, first uint32, rep bool) match {
|
||||
if best.length != 0 && best.s-best.offset == s-offset {
|
||||
// Don't retest if we have the same offset.
|
||||
return match{offset: offset, s: s}
|
||||
}
|
||||
if load32(src, offset) != first {
|
||||
return match{offset: offset, s: s}
|
||||
}
|
||||
m := match{offset: offset, s: s, length: 4 + offset, rep: rep}
|
||||
s += 4
|
||||
for s <= sLimit {
|
||||
if diff := load64(src, s) ^ load64(src, m.length); diff != 0 {
|
||||
m.length += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
m.length += 8
|
||||
}
|
||||
m.length -= offset
|
||||
m.score = score(m)
|
||||
if m.score <= -m.s {
|
||||
// Eliminate if no savings, we might find a better one.
|
||||
m.length = 0
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
bestOf := func(a, b match) match {
|
||||
if b.length == 0 {
|
||||
return a
|
||||
}
|
||||
if a.length == 0 {
|
||||
return b
|
||||
}
|
||||
as := a.score + b.s
|
||||
bs := b.score + a.s
|
||||
if as >= bs {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
best = bestOf(matchAt(getCur(candidateL), s, uint32(cv), false), matchAt(getPrev(candidateL), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getCur(candidateS), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getPrev(candidateS), s, uint32(cv), false))
|
||||
|
||||
{
|
||||
best = bestOf(best, matchAt(s-repeat+1, s+1, uint32(cv>>8), true))
|
||||
if best.length > 0 {
|
||||
// s+1
|
||||
nextShort := sTable[hash4(cv>>8, sTableBits)]
|
||||
s := s + 1
|
||||
cv := load64(src, s)
|
||||
nextLong := lTable[hash8(cv, lTableBits)]
|
||||
best = bestOf(best, matchAt(getCur(nextShort), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getPrev(nextShort), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getCur(nextLong), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getPrev(nextLong), s, uint32(cv), false))
|
||||
// Repeat at + 2
|
||||
best = bestOf(best, matchAt(s-repeat+1, s+1, uint32(cv>>8), true))
|
||||
|
||||
// s+2
|
||||
if true {
|
||||
nextShort = sTable[hash4(cv>>8, sTableBits)]
|
||||
s++
|
||||
cv = load64(src, s)
|
||||
nextLong = lTable[hash8(cv, lTableBits)]
|
||||
best = bestOf(best, matchAt(getCur(nextShort), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getPrev(nextShort), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getCur(nextLong), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getPrev(nextLong), s, uint32(cv), false))
|
||||
}
|
||||
// Search for a match at best match end, see if that is better.
|
||||
if sAt := best.s + best.length; sAt < sLimit {
|
||||
sBack := best.s
|
||||
backL := best.length
|
||||
// Load initial values
|
||||
cv = load64(src, sBack)
|
||||
// Search for mismatch
|
||||
next := lTable[hash8(load64(src, sAt), lTableBits)]
|
||||
//next := sTable[hash4(load64(src, sAt), sTableBits)]
|
||||
|
||||
if checkAt := getCur(next) - backL; checkAt > 0 {
|
||||
best = bestOf(best, matchAt(checkAt, sBack, uint32(cv), false))
|
||||
}
|
||||
if checkAt := getPrev(next) - backL; checkAt > 0 {
|
||||
best = bestOf(best, matchAt(checkAt, sBack, uint32(cv), false))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Update table
|
||||
lTable[hashL] = uint64(s) | candidateL<<32
|
||||
sTable[hashS] = uint64(s) | candidateS<<32
|
||||
|
||||
if best.length > 0 {
|
||||
break
|
||||
}
|
||||
|
||||
cv = load64(src, nextS)
|
||||
s = nextS
|
||||
}
|
||||
|
||||
// Extend backwards, not needed for repeats...
|
||||
s = best.s
|
||||
if !best.rep {
|
||||
for best.offset > 0 && s > nextEmit && src[best.offset-1] == src[s-1] {
|
||||
best.offset--
|
||||
best.length++
|
||||
s--
|
||||
}
|
||||
}
|
||||
if false && best.offset >= s {
|
||||
panic(fmt.Errorf("t %d >= s %d", best.offset, s))
|
||||
}
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+(s-nextEmit) > dstLimit {
|
||||
return 0
|
||||
}
|
||||
|
||||
base := s
|
||||
offset := s - best.offset
|
||||
|
||||
s += best.length
|
||||
|
||||
if offset > 65535 && s-base <= 5 && !best.rep {
|
||||
// Bail if the match is equal or worse to the encoding.
|
||||
s = best.s + 1
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
cv = load64(src, s)
|
||||
continue
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:base])
|
||||
if best.rep {
|
||||
if nextEmit > 0 {
|
||||
// same as `add := emitCopy(dst[d:], repeat, s-base)` but skips storing offset.
|
||||
d += emitRepeat(dst[d:], offset, best.length)
|
||||
} else {
|
||||
// First match, cannot be repeat.
|
||||
d += emitCopy(dst[d:], offset, best.length)
|
||||
}
|
||||
} else {
|
||||
d += emitCopy(dst[d:], offset, best.length)
|
||||
}
|
||||
repeat = offset
|
||||
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
if d > dstLimit {
|
||||
// Do we have space for more, if not bail.
|
||||
return 0
|
||||
}
|
||||
// Fill tables...
|
||||
for i := best.s + 1; i < s; i++ {
|
||||
cv0 := load64(src, i)
|
||||
long0 := hash8(cv0, lTableBits)
|
||||
short0 := hash4(cv0, sTableBits)
|
||||
lTable[long0] = uint64(i) | lTable[long0]<<32
|
||||
sTable[short0] = uint64(i) | sTable[short0]<<32
|
||||
}
|
||||
cv = load64(src, s)
|
||||
}
|
||||
|
||||
emitRemainder:
|
||||
if nextEmit < len(src) {
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+len(src)-nextEmit > dstLimit {
|
||||
return 0
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:])
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// encodeBlockBestSnappy encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlockBestSnappy(dst, src []byte) (d int) {
|
||||
// Initialize the hash tables.
|
||||
const (
|
||||
// Long hash matches.
|
||||
lTableBits = 19
|
||||
maxLTableSize = 1 << lTableBits
|
||||
|
||||
// Short hash matches.
|
||||
sTableBits = 16
|
||||
maxSTableSize = 1 << sTableBits
|
||||
|
||||
inputMargin = 8 + 2
|
||||
)
|
||||
|
||||
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
||||
// lets us use a fast path for emitLiteral in the main loop, while we are
|
||||
// looking for copies.
|
||||
sLimit := len(src) - inputMargin
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
return 0
|
||||
}
|
||||
|
||||
var lTable [maxLTableSize]uint64
|
||||
var sTable [maxSTableSize]uint64
|
||||
|
||||
// Bail if we can't compress to at least this.
|
||||
dstLimit := len(src) - 5
|
||||
|
||||
// nextEmit is where in src the next emitLiteral should start from.
|
||||
nextEmit := 0
|
||||
|
||||
// The encoded form must start with a literal, as there are no previous
|
||||
// bytes to copy, so we start looking for hash matches at s == 1.
|
||||
s := 1
|
||||
cv := load64(src, s)
|
||||
|
||||
// We search for a repeat at -1, but don't output repeats when nextEmit == 0
|
||||
repeat := 1
|
||||
const lowbitMask = 0xffffffff
|
||||
getCur := func(x uint64) int {
|
||||
return int(x & lowbitMask)
|
||||
}
|
||||
getPrev := func(x uint64) int {
|
||||
return int(x >> 32)
|
||||
}
|
||||
const maxSkip = 64
|
||||
|
||||
for {
|
||||
type match struct {
|
||||
offset int
|
||||
s int
|
||||
length int
|
||||
score int
|
||||
}
|
||||
var best match
|
||||
for {
|
||||
// Next src position to check
|
||||
nextS := (s-nextEmit)>>8 + 1
|
||||
if nextS > maxSkip {
|
||||
nextS = s + maxSkip
|
||||
} else {
|
||||
nextS += s
|
||||
}
|
||||
if nextS > sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
hashL := hash8(cv, lTableBits)
|
||||
hashS := hash4(cv, sTableBits)
|
||||
candidateL := lTable[hashL]
|
||||
candidateS := sTable[hashS]
|
||||
|
||||
score := func(m match) int {
|
||||
// Matches that are longer forward are penalized since we must emit it as a literal.
|
||||
score := m.length - m.s
|
||||
if nextEmit == m.s {
|
||||
// If we do not have to emit literals, we save 1 byte
|
||||
score++
|
||||
}
|
||||
offset := m.s - m.offset
|
||||
|
||||
return score - emitCopySize(offset, m.length)
|
||||
}
|
||||
|
||||
matchAt := func(offset, s int, first uint32) match {
|
||||
if best.length != 0 && best.s-best.offset == s-offset {
|
||||
// Don't retest if we have the same offset.
|
||||
return match{offset: offset, s: s}
|
||||
}
|
||||
if load32(src, offset) != first {
|
||||
return match{offset: offset, s: s}
|
||||
}
|
||||
m := match{offset: offset, s: s, length: 4 + offset}
|
||||
s += 4
|
||||
for s <= sLimit {
|
||||
if diff := load64(src, s) ^ load64(src, m.length); diff != 0 {
|
||||
m.length += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
m.length += 8
|
||||
}
|
||||
m.length -= offset
|
||||
m.score = score(m)
|
||||
if m.score <= -m.s {
|
||||
// Eliminate if no savings, we might find a better one.
|
||||
m.length = 0
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
bestOf := func(a, b match) match {
|
||||
if b.length == 0 {
|
||||
return a
|
||||
}
|
||||
if a.length == 0 {
|
||||
return b
|
||||
}
|
||||
as := a.score + b.s
|
||||
bs := b.score + a.s
|
||||
if as >= bs {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
best = bestOf(matchAt(getCur(candidateL), s, uint32(cv)), matchAt(getPrev(candidateL), s, uint32(cv)))
|
||||
best = bestOf(best, matchAt(getCur(candidateS), s, uint32(cv)))
|
||||
best = bestOf(best, matchAt(getPrev(candidateS), s, uint32(cv)))
|
||||
|
||||
{
|
||||
best = bestOf(best, matchAt(s-repeat+1, s+1, uint32(cv>>8)))
|
||||
if best.length > 0 {
|
||||
// s+1
|
||||
nextShort := sTable[hash4(cv>>8, sTableBits)]
|
||||
s := s + 1
|
||||
cv := load64(src, s)
|
||||
nextLong := lTable[hash8(cv, lTableBits)]
|
||||
best = bestOf(best, matchAt(getCur(nextShort), s, uint32(cv)))
|
||||
best = bestOf(best, matchAt(getPrev(nextShort), s, uint32(cv)))
|
||||
best = bestOf(best, matchAt(getCur(nextLong), s, uint32(cv)))
|
||||
best = bestOf(best, matchAt(getPrev(nextLong), s, uint32(cv)))
|
||||
// Repeat at + 2
|
||||
best = bestOf(best, matchAt(s-repeat+1, s+1, uint32(cv>>8)))
|
||||
|
||||
// s+2
|
||||
if true {
|
||||
nextShort = sTable[hash4(cv>>8, sTableBits)]
|
||||
s++
|
||||
cv = load64(src, s)
|
||||
nextLong = lTable[hash8(cv, lTableBits)]
|
||||
best = bestOf(best, matchAt(getCur(nextShort), s, uint32(cv)))
|
||||
best = bestOf(best, matchAt(getPrev(nextShort), s, uint32(cv)))
|
||||
best = bestOf(best, matchAt(getCur(nextLong), s, uint32(cv)))
|
||||
best = bestOf(best, matchAt(getPrev(nextLong), s, uint32(cv)))
|
||||
}
|
||||
// Search for a match at best match end, see if that is better.
|
||||
if sAt := best.s + best.length; sAt < sLimit {
|
||||
sBack := best.s
|
||||
backL := best.length
|
||||
// Load initial values
|
||||
cv = load64(src, sBack)
|
||||
// Search for mismatch
|
||||
next := lTable[hash8(load64(src, sAt), lTableBits)]
|
||||
//next := sTable[hash4(load64(src, sAt), sTableBits)]
|
||||
|
||||
if checkAt := getCur(next) - backL; checkAt > 0 {
|
||||
best = bestOf(best, matchAt(checkAt, sBack, uint32(cv)))
|
||||
}
|
||||
if checkAt := getPrev(next) - backL; checkAt > 0 {
|
||||
best = bestOf(best, matchAt(checkAt, sBack, uint32(cv)))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Update table
|
||||
lTable[hashL] = uint64(s) | candidateL<<32
|
||||
sTable[hashS] = uint64(s) | candidateS<<32
|
||||
|
||||
if best.length > 0 {
|
||||
break
|
||||
}
|
||||
|
||||
cv = load64(src, nextS)
|
||||
s = nextS
|
||||
}
|
||||
|
||||
// Extend backwards, not needed for repeats...
|
||||
s = best.s
|
||||
if true {
|
||||
for best.offset > 0 && s > nextEmit && src[best.offset-1] == src[s-1] {
|
||||
best.offset--
|
||||
best.length++
|
||||
s--
|
||||
}
|
||||
}
|
||||
if false && best.offset >= s {
|
||||
panic(fmt.Errorf("t %d >= s %d", best.offset, s))
|
||||
}
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+(s-nextEmit) > dstLimit {
|
||||
return 0
|
||||
}
|
||||
|
||||
base := s
|
||||
offset := s - best.offset
|
||||
|
||||
s += best.length
|
||||
|
||||
if offset > 65535 && s-base <= 5 {
|
||||
// Bail if the match is equal or worse to the encoding.
|
||||
s = best.s + 1
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
cv = load64(src, s)
|
||||
continue
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:base])
|
||||
d += emitCopyNoRepeat(dst[d:], offset, best.length)
|
||||
repeat = offset
|
||||
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
if d > dstLimit {
|
||||
// Do we have space for more, if not bail.
|
||||
return 0
|
||||
}
|
||||
// Fill tables...
|
||||
for i := best.s + 1; i < s; i++ {
|
||||
cv0 := load64(src, i)
|
||||
long0 := hash8(cv0, lTableBits)
|
||||
short0 := hash4(cv0, sTableBits)
|
||||
lTable[long0] = uint64(i) | lTable[long0]<<32
|
||||
sTable[short0] = uint64(i) | sTable[short0]<<32
|
||||
}
|
||||
cv = load64(src, s)
|
||||
}
|
||||
|
||||
emitRemainder:
|
||||
if nextEmit < len(src) {
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+len(src)-nextEmit > dstLimit {
|
||||
return 0
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:])
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// emitCopySize returns the size to encode the offset+length
|
||||
//
|
||||
// It assumes that:
|
||||
// 1 <= offset && offset <= math.MaxUint32
|
||||
// 4 <= length && length <= 1 << 24
|
||||
func emitCopySize(offset, length int) int {
|
||||
if offset >= 65536 {
|
||||
i := 0
|
||||
if length > 64 {
|
||||
length -= 64
|
||||
if length >= 4 {
|
||||
// Emit remaining as repeats
|
||||
return 5 + emitRepeatSize(offset, length)
|
||||
}
|
||||
i = 5
|
||||
}
|
||||
if length == 0 {
|
||||
return i
|
||||
}
|
||||
return i + 5
|
||||
}
|
||||
|
||||
// Offset no more than 2 bytes.
|
||||
if length > 64 {
|
||||
// Emit remaining as repeats, at least 4 bytes remain.
|
||||
return 3 + emitRepeatSize(offset, length-60)
|
||||
}
|
||||
if length >= 12 || offset >= 2048 {
|
||||
return 3
|
||||
}
|
||||
// Emit the remaining copy, encoded as 2 bytes.
|
||||
return 2
|
||||
}
|
||||
|
||||
// emitRepeatSize returns the number of bytes required to encode a repeat.
|
||||
// Length must be at least 4 and < 1<<24
|
||||
func emitRepeatSize(offset, length int) int {
|
||||
// Repeat offset, make length cheaper
|
||||
if length <= 4+4 || (length < 8+4 && offset < 2048) {
|
||||
return 2
|
||||
}
|
||||
if length < (1<<8)+4+4 {
|
||||
return 3
|
||||
}
|
||||
if length < (1<<16)+(1<<8)+4 {
|
||||
return 4
|
||||
}
|
||||
const maxRepeat = (1 << 24) - 1
|
||||
length -= (1 << 16) - 4
|
||||
left := 0
|
||||
if length > maxRepeat {
|
||||
left = length - maxRepeat + 4
|
||||
length = maxRepeat - 4
|
||||
}
|
||||
if left > 0 {
|
||||
return 5 + emitRepeatSize(offset, left)
|
||||
}
|
||||
return 5
|
||||
}
|
||||
431
vendor/github.com/klauspost/compress/s2/encode_better.go
сгенерированный
поставляемый
Обычный файл
431
vendor/github.com/klauspost/compress/s2/encode_better.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,431 @@
|
||||
// Copyright 2016 The Snappy-Go Authors. All rights reserved.
|
||||
// Copyright (c) 2019 Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package s2
|
||||
|
||||
import (
|
||||
"math/bits"
|
||||
)
|
||||
|
||||
// hash4 returns the hash of the lowest 4 bytes of u to fit in a hash table with h bits.
|
||||
// Preferably h should be a constant and should always be <32.
|
||||
func hash4(u uint64, h uint8) uint32 {
|
||||
const prime4bytes = 2654435761
|
||||
return (uint32(u) * prime4bytes) >> ((32 - h) & 31)
|
||||
}
|
||||
|
||||
// hash5 returns the hash of the lowest 5 bytes of u to fit in a hash table with h bits.
|
||||
// Preferably h should be a constant and should always be <64.
|
||||
func hash5(u uint64, h uint8) uint32 {
|
||||
const prime5bytes = 889523592379
|
||||
return uint32(((u << (64 - 40)) * prime5bytes) >> ((64 - h) & 63))
|
||||
}
|
||||
|
||||
// hash7 returns the hash of the lowest 7 bytes of u to fit in a hash table with h bits.
|
||||
// Preferably h should be a constant and should always be <64.
|
||||
func hash7(u uint64, h uint8) uint32 {
|
||||
const prime7bytes = 58295818150454627
|
||||
return uint32(((u << (64 - 56)) * prime7bytes) >> ((64 - h) & 63))
|
||||
}
|
||||
|
||||
// hash8 returns the hash of u to fit in a hash table with h bits.
|
||||
// Preferably h should be a constant and should always be <64.
|
||||
func hash8(u uint64, h uint8) uint32 {
|
||||
const prime8bytes = 0xcf1bbcdcb7a56463
|
||||
return uint32((u * prime8bytes) >> ((64 - h) & 63))
|
||||
}
|
||||
|
||||
// encodeBlockBetter encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlockBetterGo(dst, src []byte) (d int) {
|
||||
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
||||
// lets us use a fast path for emitLiteral in the main loop, while we are
|
||||
// looking for copies.
|
||||
sLimit := len(src) - inputMargin
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
return 0
|
||||
}
|
||||
|
||||
// Initialize the hash tables.
|
||||
const (
|
||||
// Long hash matches.
|
||||
lTableBits = 16
|
||||
maxLTableSize = 1 << lTableBits
|
||||
|
||||
// Short hash matches.
|
||||
sTableBits = 14
|
||||
maxSTableSize = 1 << sTableBits
|
||||
)
|
||||
|
||||
var lTable [maxLTableSize]uint32
|
||||
var sTable [maxSTableSize]uint32
|
||||
|
||||
// Bail if we can't compress to at least this.
|
||||
dstLimit := len(src) - len(src)>>5 - 6
|
||||
|
||||
// nextEmit is where in src the next emitLiteral should start from.
|
||||
nextEmit := 0
|
||||
|
||||
// The encoded form must start with a literal, as there are no previous
|
||||
// bytes to copy, so we start looking for hash matches at s == 1.
|
||||
s := 1
|
||||
cv := load64(src, s)
|
||||
|
||||
// We initialize repeat to 0, so we never match on first attempt
|
||||
repeat := 0
|
||||
|
||||
for {
|
||||
candidateL := 0
|
||||
nextS := 0
|
||||
for {
|
||||
// Next src position to check
|
||||
nextS = s + (s-nextEmit)>>7 + 1
|
||||
if nextS > sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
hashL := hash7(cv, lTableBits)
|
||||
hashS := hash4(cv, sTableBits)
|
||||
candidateL = int(lTable[hashL])
|
||||
candidateS := int(sTable[hashS])
|
||||
lTable[hashL] = uint32(s)
|
||||
sTable[hashS] = uint32(s)
|
||||
|
||||
// Check repeat at offset checkRep.
|
||||
const checkRep = 1
|
||||
if false && uint32(cv>>(checkRep*8)) == load32(src, s-repeat+checkRep) {
|
||||
base := s + checkRep
|
||||
// Extend back
|
||||
for i := base - repeat; base > nextEmit && i > 0 && src[i-1] == src[base-1]; {
|
||||
i--
|
||||
base--
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:base])
|
||||
|
||||
// Extend forward
|
||||
candidate := s - repeat + 4 + checkRep
|
||||
s += 4 + checkRep
|
||||
for s < len(src) {
|
||||
if len(src)-s < 8 {
|
||||
if src[s] == src[candidate] {
|
||||
s++
|
||||
candidate++
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if diff := load64(src, s) ^ load64(src, candidate); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
if nextEmit > 0 {
|
||||
// same as `add := emitCopy(dst[d:], repeat, s-base)` but skips storing offset.
|
||||
d += emitRepeat(dst[d:], repeat, s-base)
|
||||
} else {
|
||||
// First match, cannot be repeat.
|
||||
d += emitCopy(dst[d:], repeat, s-base)
|
||||
}
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
cv = load64(src, s)
|
||||
continue
|
||||
}
|
||||
|
||||
if uint32(cv) == load32(src, candidateL) {
|
||||
break
|
||||
}
|
||||
|
||||
// Check our short candidate
|
||||
if uint32(cv) == load32(src, candidateS) {
|
||||
// Try a long candidate at s+1
|
||||
hashL = hash7(cv>>8, lTableBits)
|
||||
candidateL = int(lTable[hashL])
|
||||
lTable[hashL] = uint32(s + 1)
|
||||
if uint32(cv>>8) == load32(src, candidateL) {
|
||||
s++
|
||||
break
|
||||
}
|
||||
// Use our short candidate.
|
||||
candidateL = candidateS
|
||||
break
|
||||
}
|
||||
|
||||
cv = load64(src, nextS)
|
||||
s = nextS
|
||||
}
|
||||
|
||||
// Extend backwards
|
||||
for candidateL > 0 && s > nextEmit && src[candidateL-1] == src[s-1] {
|
||||
candidateL--
|
||||
s--
|
||||
}
|
||||
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+(s-nextEmit) > dstLimit {
|
||||
return 0
|
||||
}
|
||||
|
||||
base := s
|
||||
offset := base - candidateL
|
||||
|
||||
// Extend the 4-byte match as long as possible.
|
||||
s += 4
|
||||
candidateL += 4
|
||||
for s < len(src) {
|
||||
if len(src)-s < 8 {
|
||||
if src[s] == src[candidateL] {
|
||||
s++
|
||||
candidateL++
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if diff := load64(src, s) ^ load64(src, candidateL); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidateL += 8
|
||||
}
|
||||
|
||||
if offset > 65535 && s-base <= 5 && repeat != offset {
|
||||
// Bail if the match is equal or worse to the encoding.
|
||||
s = nextS + 1
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
cv = load64(src, s)
|
||||
continue
|
||||
}
|
||||
|
||||
d += emitLiteral(dst[d:], src[nextEmit:base])
|
||||
if repeat == offset {
|
||||
d += emitRepeat(dst[d:], offset, s-base)
|
||||
} else {
|
||||
d += emitCopy(dst[d:], offset, s-base)
|
||||
repeat = offset
|
||||
}
|
||||
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
if d > dstLimit {
|
||||
// Do we have space for more, if not bail.
|
||||
return 0
|
||||
}
|
||||
// Index match start+1 (long) and start+2 (short)
|
||||
index0 := base + 1
|
||||
// Index match end-2 (long) and end-1 (short)
|
||||
index1 := s - 2
|
||||
|
||||
cv0 := load64(src, index0)
|
||||
cv1 := load64(src, index1)
|
||||
cv = load64(src, s)
|
||||
lTable[hash7(cv0, lTableBits)] = uint32(index0)
|
||||
lTable[hash7(cv0>>8, lTableBits)] = uint32(index0 + 1)
|
||||
lTable[hash7(cv1, lTableBits)] = uint32(index1)
|
||||
lTable[hash7(cv1>>8, lTableBits)] = uint32(index1 + 1)
|
||||
sTable[hash4(cv0>>8, sTableBits)] = uint32(index0 + 1)
|
||||
sTable[hash4(cv0>>16, sTableBits)] = uint32(index0 + 2)
|
||||
sTable[hash4(cv1>>8, sTableBits)] = uint32(index1 + 1)
|
||||
}
|
||||
|
||||
emitRemainder:
|
||||
if nextEmit < len(src) {
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+len(src)-nextEmit > dstLimit {
|
||||
return 0
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:])
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// encodeBlockBetterSnappyGo encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlockBetterSnappyGo(dst, src []byte) (d int) {
|
||||
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
||||
// lets us use a fast path for emitLiteral in the main loop, while we are
|
||||
// looking for copies.
|
||||
sLimit := len(src) - inputMargin
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
return 0
|
||||
}
|
||||
|
||||
// Initialize the hash tables.
|
||||
const (
|
||||
// Long hash matches.
|
||||
lTableBits = 16
|
||||
maxLTableSize = 1 << lTableBits
|
||||
|
||||
// Short hash matches.
|
||||
sTableBits = 14
|
||||
maxSTableSize = 1 << sTableBits
|
||||
)
|
||||
|
||||
var lTable [maxLTableSize]uint32
|
||||
var sTable [maxSTableSize]uint32
|
||||
|
||||
// Bail if we can't compress to at least this.
|
||||
dstLimit := len(src) - len(src)>>5 - 6
|
||||
|
||||
// nextEmit is where in src the next emitLiteral should start from.
|
||||
nextEmit := 0
|
||||
|
||||
// The encoded form must start with a literal, as there are no previous
|
||||
// bytes to copy, so we start looking for hash matches at s == 1.
|
||||
s := 1
|
||||
cv := load64(src, s)
|
||||
|
||||
// We initialize repeat to 0, so we never match on first attempt
|
||||
repeat := 0
|
||||
const maxSkip = 100
|
||||
|
||||
for {
|
||||
candidateL := 0
|
||||
nextS := 0
|
||||
for {
|
||||
// Next src position to check
|
||||
nextS = (s-nextEmit)>>7 + 1
|
||||
if nextS > maxSkip {
|
||||
nextS = s + maxSkip
|
||||
} else {
|
||||
nextS += s
|
||||
}
|
||||
|
||||
if nextS > sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
hashL := hash7(cv, lTableBits)
|
||||
hashS := hash4(cv, sTableBits)
|
||||
candidateL = int(lTable[hashL])
|
||||
candidateS := int(sTable[hashS])
|
||||
lTable[hashL] = uint32(s)
|
||||
sTable[hashS] = uint32(s)
|
||||
|
||||
if uint32(cv) == load32(src, candidateL) {
|
||||
break
|
||||
}
|
||||
|
||||
// Check our short candidate
|
||||
if uint32(cv) == load32(src, candidateS) {
|
||||
// Try a long candidate at s+1
|
||||
hashL = hash7(cv>>8, lTableBits)
|
||||
candidateL = int(lTable[hashL])
|
||||
lTable[hashL] = uint32(s + 1)
|
||||
if uint32(cv>>8) == load32(src, candidateL) {
|
||||
s++
|
||||
break
|
||||
}
|
||||
// Use our short candidate.
|
||||
candidateL = candidateS
|
||||
break
|
||||
}
|
||||
|
||||
cv = load64(src, nextS)
|
||||
s = nextS
|
||||
}
|
||||
|
||||
// Extend backwards
|
||||
for candidateL > 0 && s > nextEmit && src[candidateL-1] == src[s-1] {
|
||||
candidateL--
|
||||
s--
|
||||
}
|
||||
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+(s-nextEmit) > dstLimit {
|
||||
return 0
|
||||
}
|
||||
|
||||
base := s
|
||||
offset := base - candidateL
|
||||
|
||||
// Extend the 4-byte match as long as possible.
|
||||
s += 4
|
||||
candidateL += 4
|
||||
for s < len(src) {
|
||||
if len(src)-s < 8 {
|
||||
if src[s] == src[candidateL] {
|
||||
s++
|
||||
candidateL++
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if diff := load64(src, s) ^ load64(src, candidateL); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidateL += 8
|
||||
}
|
||||
|
||||
if offset > 65535 && s-base <= 5 && repeat != offset {
|
||||
// Bail if the match is equal or worse to the encoding.
|
||||
s = nextS + 1
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
cv = load64(src, s)
|
||||
continue
|
||||
}
|
||||
|
||||
d += emitLiteral(dst[d:], src[nextEmit:base])
|
||||
d += emitCopyNoRepeat(dst[d:], offset, s-base)
|
||||
repeat = offset
|
||||
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
if d > dstLimit {
|
||||
// Do we have space for more, if not bail.
|
||||
return 0
|
||||
}
|
||||
// Index match start+1 (long) and start+2 (short)
|
||||
index0 := base + 1
|
||||
// Index match end-2 (long) and end-1 (short)
|
||||
index1 := s - 2
|
||||
|
||||
cv0 := load64(src, index0)
|
||||
cv1 := load64(src, index1)
|
||||
cv = load64(src, s)
|
||||
lTable[hash7(cv0, lTableBits)] = uint32(index0)
|
||||
lTable[hash7(cv0>>8, lTableBits)] = uint32(index0 + 1)
|
||||
lTable[hash7(cv1, lTableBits)] = uint32(index1)
|
||||
lTable[hash7(cv1>>8, lTableBits)] = uint32(index1 + 1)
|
||||
sTable[hash4(cv0>>8, sTableBits)] = uint32(index0 + 1)
|
||||
sTable[hash4(cv0>>16, sTableBits)] = uint32(index0 + 2)
|
||||
sTable[hash4(cv1>>8, sTableBits)] = uint32(index1 + 1)
|
||||
}
|
||||
|
||||
emitRemainder:
|
||||
if nextEmit < len(src) {
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+len(src)-nextEmit > dstLimit {
|
||||
return 0
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:])
|
||||
}
|
||||
return d
|
||||
}
|
||||
298
vendor/github.com/klauspost/compress/s2/encode_go.go
сгенерированный
поставляемый
Обычный файл
298
vendor/github.com/klauspost/compress/s2/encode_go.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,298 @@
|
||||
//go:build !amd64 || appengine || !gc || noasm
|
||||
// +build !amd64 appengine !gc noasm
|
||||
|
||||
package s2
|
||||
|
||||
import (
|
||||
"math/bits"
|
||||
)
|
||||
|
||||
// encodeBlock encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
// len(dst) >= MaxEncodedLen(len(src))
|
||||
func encodeBlock(dst, src []byte) (d int) {
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
return 0
|
||||
}
|
||||
return encodeBlockGo(dst, src)
|
||||
}
|
||||
|
||||
// encodeBlockBetter encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
// len(dst) >= MaxEncodedLen(len(src))
|
||||
func encodeBlockBetter(dst, src []byte) (d int) {
|
||||
return encodeBlockBetterGo(dst, src)
|
||||
}
|
||||
|
||||
// encodeBlockBetter encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
// len(dst) >= MaxEncodedLen(len(src))
|
||||
func encodeBlockBetterSnappy(dst, src []byte) (d int) {
|
||||
return encodeBlockBetterSnappyGo(dst, src)
|
||||
}
|
||||
|
||||
// encodeBlock encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
// len(dst) >= MaxEncodedLen(len(src))
|
||||
func encodeBlockSnappy(dst, src []byte) (d int) {
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
return 0
|
||||
}
|
||||
return encodeBlockSnappyGo(dst, src)
|
||||
}
|
||||
|
||||
// emitLiteral writes a literal chunk and returns the number of bytes written.
|
||||
//
|
||||
// It assumes that:
|
||||
// dst is long enough to hold the encoded bytes
|
||||
// 0 <= len(lit) && len(lit) <= math.MaxUint32
|
||||
func emitLiteral(dst, lit []byte) int {
|
||||
if len(lit) == 0 {
|
||||
return 0
|
||||
}
|
||||
const num = 63<<2 | tagLiteral
|
||||
i, n := 0, uint(len(lit)-1)
|
||||
switch {
|
||||
case n < 60:
|
||||
dst[0] = uint8(n)<<2 | tagLiteral
|
||||
i = 1
|
||||
case n < 1<<8:
|
||||
dst[1] = uint8(n)
|
||||
dst[0] = 60<<2 | tagLiteral
|
||||
i = 2
|
||||
case n < 1<<16:
|
||||
dst[2] = uint8(n >> 8)
|
||||
dst[1] = uint8(n)
|
||||
dst[0] = 61<<2 | tagLiteral
|
||||
i = 3
|
||||
case n < 1<<24:
|
||||
dst[3] = uint8(n >> 16)
|
||||
dst[2] = uint8(n >> 8)
|
||||
dst[1] = uint8(n)
|
||||
dst[0] = 62<<2 | tagLiteral
|
||||
i = 4
|
||||
default:
|
||||
dst[4] = uint8(n >> 24)
|
||||
dst[3] = uint8(n >> 16)
|
||||
dst[2] = uint8(n >> 8)
|
||||
dst[1] = uint8(n)
|
||||
dst[0] = 63<<2 | tagLiteral
|
||||
i = 5
|
||||
}
|
||||
return i + copy(dst[i:], lit)
|
||||
}
|
||||
|
||||
// emitRepeat writes a repeat chunk and returns the number of bytes written.
|
||||
// Length must be at least 4 and < 1<<24
|
||||
func emitRepeat(dst []byte, offset, length int) int {
|
||||
// Repeat offset, make length cheaper
|
||||
length -= 4
|
||||
if length <= 4 {
|
||||
dst[0] = uint8(length)<<2 | tagCopy1
|
||||
dst[1] = 0
|
||||
return 2
|
||||
}
|
||||
if length < 8 && offset < 2048 {
|
||||
// Encode WITH offset
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(offset>>8)<<5 | uint8(length)<<2 | tagCopy1
|
||||
return 2
|
||||
}
|
||||
if length < (1<<8)+4 {
|
||||
length -= 4
|
||||
dst[2] = uint8(length)
|
||||
dst[1] = 0
|
||||
dst[0] = 5<<2 | tagCopy1
|
||||
return 3
|
||||
}
|
||||
if length < (1<<16)+(1<<8) {
|
||||
length -= 1 << 8
|
||||
dst[3] = uint8(length >> 8)
|
||||
dst[2] = uint8(length >> 0)
|
||||
dst[1] = 0
|
||||
dst[0] = 6<<2 | tagCopy1
|
||||
return 4
|
||||
}
|
||||
const maxRepeat = (1 << 24) - 1
|
||||
length -= 1 << 16
|
||||
left := 0
|
||||
if length > maxRepeat {
|
||||
left = length - maxRepeat + 4
|
||||
length = maxRepeat - 4
|
||||
}
|
||||
dst[4] = uint8(length >> 16)
|
||||
dst[3] = uint8(length >> 8)
|
||||
dst[2] = uint8(length >> 0)
|
||||
dst[1] = 0
|
||||
dst[0] = 7<<2 | tagCopy1
|
||||
if left > 0 {
|
||||
return 5 + emitRepeat(dst[5:], offset, left)
|
||||
}
|
||||
return 5
|
||||
}
|
||||
|
||||
// emitCopy writes a copy chunk and returns the number of bytes written.
|
||||
//
|
||||
// It assumes that:
|
||||
// dst is long enough to hold the encoded bytes
|
||||
// 1 <= offset && offset <= math.MaxUint32
|
||||
// 4 <= length && length <= 1 << 24
|
||||
func emitCopy(dst []byte, offset, length int) int {
|
||||
if offset >= 65536 {
|
||||
i := 0
|
||||
if length > 64 {
|
||||
// Emit a length 64 copy, encoded as 5 bytes.
|
||||
dst[4] = uint8(offset >> 24)
|
||||
dst[3] = uint8(offset >> 16)
|
||||
dst[2] = uint8(offset >> 8)
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = 63<<2 | tagCopy4
|
||||
length -= 64
|
||||
if length >= 4 {
|
||||
// Emit remaining as repeats
|
||||
return 5 + emitRepeat(dst[5:], offset, length)
|
||||
}
|
||||
i = 5
|
||||
}
|
||||
if length == 0 {
|
||||
return i
|
||||
}
|
||||
// Emit a copy, offset encoded as 4 bytes.
|
||||
dst[i+0] = uint8(length-1)<<2 | tagCopy4
|
||||
dst[i+1] = uint8(offset)
|
||||
dst[i+2] = uint8(offset >> 8)
|
||||
dst[i+3] = uint8(offset >> 16)
|
||||
dst[i+4] = uint8(offset >> 24)
|
||||
return i + 5
|
||||
}
|
||||
|
||||
// Offset no more than 2 bytes.
|
||||
if length > 64 {
|
||||
// Emit a length 60 copy, encoded as 3 bytes.
|
||||
// Emit remaining as repeat value (minimum 4 bytes).
|
||||
dst[2] = uint8(offset >> 8)
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = 59<<2 | tagCopy2
|
||||
length -= 60
|
||||
// Emit remaining as repeats, at least 4 bytes remain.
|
||||
return 3 + emitRepeat(dst[3:], offset, length)
|
||||
}
|
||||
if length >= 12 || offset >= 2048 {
|
||||
// Emit the remaining copy, encoded as 3 bytes.
|
||||
dst[2] = uint8(offset >> 8)
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(length-1)<<2 | tagCopy2
|
||||
return 3
|
||||
}
|
||||
// Emit the remaining copy, encoded as 2 bytes.
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(offset>>8)<<5 | uint8(length-4)<<2 | tagCopy1
|
||||
return 2
|
||||
}
|
||||
|
||||
// emitCopyNoRepeat writes a copy chunk and returns the number of bytes written.
|
||||
//
|
||||
// It assumes that:
|
||||
// dst is long enough to hold the encoded bytes
|
||||
// 1 <= offset && offset <= math.MaxUint32
|
||||
// 4 <= length && length <= 1 << 24
|
||||
func emitCopyNoRepeat(dst []byte, offset, length int) int {
|
||||
if offset >= 65536 {
|
||||
i := 0
|
||||
if length > 64 {
|
||||
// Emit a length 64 copy, encoded as 5 bytes.
|
||||
dst[4] = uint8(offset >> 24)
|
||||
dst[3] = uint8(offset >> 16)
|
||||
dst[2] = uint8(offset >> 8)
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = 63<<2 | tagCopy4
|
||||
length -= 64
|
||||
if length >= 4 {
|
||||
// Emit remaining as repeats
|
||||
return 5 + emitCopyNoRepeat(dst[5:], offset, length)
|
||||
}
|
||||
i = 5
|
||||
}
|
||||
if length == 0 {
|
||||
return i
|
||||
}
|
||||
// Emit a copy, offset encoded as 4 bytes.
|
||||
dst[i+0] = uint8(length-1)<<2 | tagCopy4
|
||||
dst[i+1] = uint8(offset)
|
||||
dst[i+2] = uint8(offset >> 8)
|
||||
dst[i+3] = uint8(offset >> 16)
|
||||
dst[i+4] = uint8(offset >> 24)
|
||||
return i + 5
|
||||
}
|
||||
|
||||
// Offset no more than 2 bytes.
|
||||
if length > 64 {
|
||||
// Emit a length 60 copy, encoded as 3 bytes.
|
||||
// Emit remaining as repeat value (minimum 4 bytes).
|
||||
dst[2] = uint8(offset >> 8)
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = 59<<2 | tagCopy2
|
||||
length -= 60
|
||||
// Emit remaining as repeats, at least 4 bytes remain.
|
||||
return 3 + emitCopyNoRepeat(dst[3:], offset, length)
|
||||
}
|
||||
if length >= 12 || offset >= 2048 {
|
||||
// Emit the remaining copy, encoded as 3 bytes.
|
||||
dst[2] = uint8(offset >> 8)
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(length-1)<<2 | tagCopy2
|
||||
return 3
|
||||
}
|
||||
// Emit the remaining copy, encoded as 2 bytes.
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(offset>>8)<<5 | uint8(length-4)<<2 | tagCopy1
|
||||
return 2
|
||||
}
|
||||
|
||||
// matchLen returns how many bytes match in a and b
|
||||
//
|
||||
// It assumes that:
|
||||
// len(a) <= len(b)
|
||||
//
|
||||
func matchLen(a []byte, b []byte) int {
|
||||
b = b[:len(a)]
|
||||
var checked int
|
||||
if len(a) > 4 {
|
||||
// Try 4 bytes first
|
||||
if diff := load32(a, 0) ^ load32(b, 0); diff != 0 {
|
||||
return bits.TrailingZeros32(diff) >> 3
|
||||
}
|
||||
// Switch to 8 byte matching.
|
||||
checked = 4
|
||||
a = a[4:]
|
||||
b = b[4:]
|
||||
for len(a) >= 8 {
|
||||
b = b[:len(a)]
|
||||
if diff := load64(a, 0) ^ load64(b, 0); diff != 0 {
|
||||
return checked + (bits.TrailingZeros64(diff) >> 3)
|
||||
}
|
||||
checked += 8
|
||||
a = a[8:]
|
||||
b = b[8:]
|
||||
}
|
||||
}
|
||||
b = b[:len(a)]
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
return int(i) + checked
|
||||
}
|
||||
}
|
||||
return len(a) + checked
|
||||
}
|
||||
189
vendor/github.com/klauspost/compress/s2/encodeblock_amd64.go
сгенерированный
поставляемый
Обычный файл
189
vendor/github.com/klauspost/compress/s2/encodeblock_amd64.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,189 @@
|
||||
// Code generated by command: go run gen.go -out ../encodeblock_amd64.s -stubs ../encodeblock_amd64.go -pkg=s2. DO NOT EDIT.
|
||||
|
||||
//go:build !appengine && !noasm && gc
|
||||
// +build !appengine,!noasm,gc
|
||||
|
||||
package s2
|
||||
|
||||
// encodeBlockAsm encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 4294967295 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeBlockAsm(dst []byte, src []byte) int
|
||||
|
||||
// encodeBlockAsm4MB encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 4194304 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeBlockAsm4MB(dst []byte, src []byte) int
|
||||
|
||||
// encodeBlockAsm12B encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 16383 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeBlockAsm12B(dst []byte, src []byte) int
|
||||
|
||||
// encodeBlockAsm10B encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 4095 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeBlockAsm10B(dst []byte, src []byte) int
|
||||
|
||||
// encodeBlockAsm8B encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 511 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeBlockAsm8B(dst []byte, src []byte) int
|
||||
|
||||
// encodeBetterBlockAsm encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 4294967295 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeBetterBlockAsm(dst []byte, src []byte) int
|
||||
|
||||
// encodeBetterBlockAsm4MB encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 4194304 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeBetterBlockAsm4MB(dst []byte, src []byte) int
|
||||
|
||||
// encodeBetterBlockAsm12B encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 16383 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeBetterBlockAsm12B(dst []byte, src []byte) int
|
||||
|
||||
// encodeBetterBlockAsm10B encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 4095 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeBetterBlockAsm10B(dst []byte, src []byte) int
|
||||
|
||||
// encodeBetterBlockAsm8B encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 511 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeBetterBlockAsm8B(dst []byte, src []byte) int
|
||||
|
||||
// encodeSnappyBlockAsm encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 4294967295 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeSnappyBlockAsm(dst []byte, src []byte) int
|
||||
|
||||
// encodeSnappyBlockAsm64K encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 65535 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeSnappyBlockAsm64K(dst []byte, src []byte) int
|
||||
|
||||
// encodeSnappyBlockAsm12B encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 16383 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeSnappyBlockAsm12B(dst []byte, src []byte) int
|
||||
|
||||
// encodeSnappyBlockAsm10B encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 4095 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeSnappyBlockAsm10B(dst []byte, src []byte) int
|
||||
|
||||
// encodeSnappyBlockAsm8B encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 511 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeSnappyBlockAsm8B(dst []byte, src []byte) int
|
||||
|
||||
// encodeSnappyBetterBlockAsm encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 4294967295 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeSnappyBetterBlockAsm(dst []byte, src []byte) int
|
||||
|
||||
// encodeSnappyBetterBlockAsm64K encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 65535 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeSnappyBetterBlockAsm64K(dst []byte, src []byte) int
|
||||
|
||||
// encodeSnappyBetterBlockAsm12B encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 16383 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeSnappyBetterBlockAsm12B(dst []byte, src []byte) int
|
||||
|
||||
// encodeSnappyBetterBlockAsm10B encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 4095 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeSnappyBetterBlockAsm10B(dst []byte, src []byte) int
|
||||
|
||||
// encodeSnappyBetterBlockAsm8B encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 511 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeSnappyBetterBlockAsm8B(dst []byte, src []byte) int
|
||||
|
||||
// emitLiteral writes a literal chunk and returns the number of bytes written.
|
||||
//
|
||||
// It assumes that:
|
||||
// dst is long enough to hold the encoded bytes with margin of 0 bytes
|
||||
// 0 <= len(lit) && len(lit) <= math.MaxUint32
|
||||
//
|
||||
//go:noescape
|
||||
func emitLiteral(dst []byte, lit []byte) int
|
||||
|
||||
// emitRepeat writes a repeat chunk and returns the number of bytes written.
|
||||
// Length must be at least 4 and < 1<<32
|
||||
//
|
||||
//go:noescape
|
||||
func emitRepeat(dst []byte, offset int, length int) int
|
||||
|
||||
// emitCopy writes a copy chunk and returns the number of bytes written.
|
||||
//
|
||||
// It assumes that:
|
||||
// dst is long enough to hold the encoded bytes
|
||||
// 1 <= offset && offset <= math.MaxUint32
|
||||
// 4 <= length && length <= 1 << 24
|
||||
//
|
||||
//go:noescape
|
||||
func emitCopy(dst []byte, offset int, length int) int
|
||||
|
||||
// emitCopyNoRepeat writes a copy chunk and returns the number of bytes written.
|
||||
//
|
||||
// It assumes that:
|
||||
// dst is long enough to hold the encoded bytes
|
||||
// 1 <= offset && offset <= math.MaxUint32
|
||||
// 4 <= length && length <= 1 << 24
|
||||
//
|
||||
//go:noescape
|
||||
func emitCopyNoRepeat(dst []byte, offset int, length int) int
|
||||
|
||||
// matchLen returns how many bytes match in a and b
|
||||
//
|
||||
// It assumes that:
|
||||
// len(a) <= len(b)
|
||||
//
|
||||
//go:noescape
|
||||
func matchLen(a []byte, b []byte) int
|
||||
15678
vendor/github.com/klauspost/compress/s2/encodeblock_amd64.s
сгенерированный
поставляемый
Обычный файл
15678
vendor/github.com/klauspost/compress/s2/encodeblock_amd64.s
сгенерированный
поставляемый
Обычный файл
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
139
vendor/github.com/klauspost/compress/s2/s2.go
сгенерированный
поставляемый
Обычный файл
139
vendor/github.com/klauspost/compress/s2/s2.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,139 @@
|
||||
// Copyright 2011 The Snappy-Go Authors. All rights reserved.
|
||||
// Copyright (c) 2019 Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Package s2 implements the S2 compression format.
|
||||
//
|
||||
// S2 is an extension of Snappy. Similar to Snappy S2 is aimed for high throughput,
|
||||
// which is why it features concurrent compression for bigger payloads.
|
||||
//
|
||||
// Decoding is compatible with Snappy compressed content,
|
||||
// but content compressed with S2 cannot be decompressed by Snappy.
|
||||
//
|
||||
// For more information on Snappy/S2 differences see README in: https://github.com/klauspost/compress/tree/master/s2
|
||||
//
|
||||
// There are actually two S2 formats: block and stream. They are related,
|
||||
// but different: trying to decompress block-compressed data as a S2 stream
|
||||
// will fail, and vice versa. The block format is the Decode and Encode
|
||||
// functions and the stream format is the Reader and Writer types.
|
||||
//
|
||||
// A "better" compression option is available. This will trade some compression
|
||||
// speed
|
||||
//
|
||||
// The block format, the more common case, is used when the complete size (the
|
||||
// number of bytes) of the original data is known upfront, at the time
|
||||
// compression starts. The stream format, also known as the framing format, is
|
||||
// for when that isn't always true.
|
||||
//
|
||||
// Blocks to not offer much data protection, so it is up to you to
|
||||
// add data validation of decompressed blocks.
|
||||
//
|
||||
// Streams perform CRC validation of the decompressed data.
|
||||
// Stream compression will also be performed on multiple CPU cores concurrently
|
||||
// significantly improving throughput.
|
||||
package s2
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"hash/crc32"
|
||||
)
|
||||
|
||||
/*
|
||||
Each encoded block begins with the varint-encoded length of the decoded data,
|
||||
followed by a sequence of chunks. Chunks begin and end on byte boundaries. The
|
||||
first byte of each chunk is broken into its 2 least and 6 most significant bits
|
||||
called l and m: l ranges in [0, 4) and m ranges in [0, 64). l is the chunk tag.
|
||||
Zero means a literal tag. All other values mean a copy tag.
|
||||
|
||||
For literal tags:
|
||||
- If m < 60, the next 1 + m bytes are literal bytes.
|
||||
- Otherwise, let n be the little-endian unsigned integer denoted by the next
|
||||
m - 59 bytes. The next 1 + n bytes after that are literal bytes.
|
||||
|
||||
For copy tags, length bytes are copied from offset bytes ago, in the style of
|
||||
Lempel-Ziv compression algorithms. In particular:
|
||||
- For l == 1, the offset ranges in [0, 1<<11) and the length in [4, 12).
|
||||
The length is 4 + the low 3 bits of m. The high 3 bits of m form bits 8-10
|
||||
of the offset. The next byte is bits 0-7 of the offset.
|
||||
- For l == 2, the offset ranges in [0, 1<<16) and the length in [1, 65).
|
||||
The length is 1 + m. The offset is the little-endian unsigned integer
|
||||
denoted by the next 2 bytes.
|
||||
- For l == 3, the offset ranges in [0, 1<<32) and the length in
|
||||
[1, 65). The length is 1 + m. The offset is the little-endian unsigned
|
||||
integer denoted by the next 4 bytes.
|
||||
*/
|
||||
const (
|
||||
tagLiteral = 0x00
|
||||
tagCopy1 = 0x01
|
||||
tagCopy2 = 0x02
|
||||
tagCopy4 = 0x03
|
||||
)
|
||||
|
||||
const (
|
||||
checksumSize = 4
|
||||
chunkHeaderSize = 4
|
||||
magicChunk = "\xff\x06\x00\x00" + magicBody
|
||||
magicChunkSnappy = "\xff\x06\x00\x00" + magicBodySnappy
|
||||
magicBodySnappy = "sNaPpY"
|
||||
magicBody = "S2sTwO"
|
||||
|
||||
// maxBlockSize is the maximum size of the input to encodeBlock.
|
||||
//
|
||||
// For the framing format (Writer type instead of Encode function),
|
||||
// this is the maximum uncompressed size of a block.
|
||||
maxBlockSize = 4 << 20
|
||||
|
||||
// minBlockSize is the minimum size of block setting when creating a writer.
|
||||
minBlockSize = 4 << 10
|
||||
|
||||
// Default block size
|
||||
defaultBlockSize = 1 << 20
|
||||
|
||||
// maxSnappyBlockSize is the maximum snappy block size.
|
||||
maxSnappyBlockSize = 1 << 16
|
||||
|
||||
obufHeaderLen = checksumSize + chunkHeaderSize
|
||||
)
|
||||
|
||||
const (
|
||||
chunkTypeCompressedData = 0x00
|
||||
chunkTypeUncompressedData = 0x01
|
||||
chunkTypePadding = 0xfe
|
||||
chunkTypeStreamIdentifier = 0xff
|
||||
)
|
||||
|
||||
var crcTable = crc32.MakeTable(crc32.Castagnoli)
|
||||
|
||||
// crc implements the checksum specified in section 3 of
|
||||
// https://github.com/google/snappy/blob/master/framing_format.txt
|
||||
func crc(b []byte) uint32 {
|
||||
c := crc32.Update(0, crcTable, b)
|
||||
return c>>15 | c<<17 + 0xa282ead8
|
||||
}
|
||||
|
||||
// literalExtraSize returns the extra size of encoding n literals.
|
||||
// n should be >= 0 and <= math.MaxUint32.
|
||||
func literalExtraSize(n int64) int64 {
|
||||
if n == 0 {
|
||||
return 0
|
||||
}
|
||||
switch {
|
||||
case n < 60:
|
||||
return 1
|
||||
case n < 1<<8:
|
||||
return 2
|
||||
case n < 1<<16:
|
||||
return 3
|
||||
case n < 1<<24:
|
||||
return 4
|
||||
default:
|
||||
return 5
|
||||
}
|
||||
}
|
||||
|
||||
type byter interface {
|
||||
Bytes() []byte
|
||||
}
|
||||
|
||||
var _ byter = &bytes.Buffer{}
|
||||
886
vendor/github.com/klauspost/compress/zip/reader.go
сгенерированный
поставляемый
Обычный файл
886
vendor/github.com/klauspost/compress/zip/reader.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,886 @@
|
||||
// Copyright 2010 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
//go:build go1.16
|
||||
// +build go1.16
|
||||
|
||||
package zip
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"hash"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrFormat = errors.New("zip: not a valid zip file")
|
||||
ErrAlgorithm = errors.New("zip: unsupported compression algorithm")
|
||||
ErrChecksum = errors.New("zip: checksum error")
|
||||
)
|
||||
|
||||
// A Reader serves content from a ZIP archive.
|
||||
type Reader struct {
|
||||
r io.ReaderAt
|
||||
File []*File
|
||||
Comment string
|
||||
decompressors map[uint16]Decompressor
|
||||
|
||||
// fileList is a list of files sorted by ename,
|
||||
// for use by the Open method.
|
||||
fileListOnce sync.Once
|
||||
fileList []fileListEntry
|
||||
}
|
||||
|
||||
// A ReadCloser is a Reader that must be closed when no longer needed.
|
||||
type ReadCloser struct {
|
||||
f *os.File
|
||||
Reader
|
||||
}
|
||||
|
||||
// A File is a single file in a ZIP archive.
|
||||
// The file information is in the embedded FileHeader.
|
||||
// The file content can be accessed by calling Open.
|
||||
type File struct {
|
||||
FileHeader
|
||||
zip *Reader
|
||||
zipr io.ReaderAt
|
||||
headerOffset int64
|
||||
zip64 bool // zip64 extended information extra field presence
|
||||
descErr error // error reading the data descriptor during init
|
||||
}
|
||||
|
||||
// OpenReader will open the Zip file specified by name and return a ReadCloser.
|
||||
func OpenReader(name string) (*ReadCloser, error) {
|
||||
f, err := os.Open(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fi, err := f.Stat()
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return nil, err
|
||||
}
|
||||
r := new(ReadCloser)
|
||||
if err := r.init(f, fi.Size()); err != nil {
|
||||
f.Close()
|
||||
return nil, err
|
||||
}
|
||||
r.f = f
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// NewReader returns a new Reader reading from r, which is assumed to
|
||||
// have the given size in bytes.
|
||||
func NewReader(r io.ReaderAt, size int64) (*Reader, error) {
|
||||
if size < 0 {
|
||||
return nil, errors.New("zip: size cannot be negative")
|
||||
}
|
||||
zr := new(Reader)
|
||||
if err := zr.init(r, size); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return zr, nil
|
||||
}
|
||||
|
||||
func (z *Reader) init(r io.ReaderAt, size int64) error {
|
||||
end, err := readDirectoryEnd(r, size)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
z.r = r
|
||||
// Since the number of directory records is not validated, it is not
|
||||
// safe to preallocate z.File without first checking that the specified
|
||||
// number of files is reasonable, since a malformed archive may
|
||||
// indicate it contains up to 1 << 128 - 1 files. Since each file has a
|
||||
// header which will be _at least_ 30 bytes we can safely preallocate
|
||||
// if (data size / 30) >= end.directoryRecords.
|
||||
if (uint64(size)-end.directorySize)/30 >= end.directoryRecords {
|
||||
z.File = make([]*File, 0, end.directoryRecords)
|
||||
}
|
||||
z.Comment = end.comment
|
||||
rs := io.NewSectionReader(r, 0, size)
|
||||
if _, err = rs.Seek(int64(end.directoryOffset), io.SeekStart); err != nil {
|
||||
return err
|
||||
}
|
||||
buf := bufio.NewReader(rs)
|
||||
|
||||
// The count of files inside a zip is truncated to fit in a uint16.
|
||||
// Gloss over this by reading headers until we encounter
|
||||
// a bad one, and then only report an ErrFormat or UnexpectedEOF if
|
||||
// the file count modulo 65536 is incorrect.
|
||||
for {
|
||||
f := &File{zip: z, zipr: r}
|
||||
err = readDirectoryHeader(f, buf)
|
||||
if err == ErrFormat || err == io.ErrUnexpectedEOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f.readDataDescriptor()
|
||||
z.File = append(z.File, f)
|
||||
}
|
||||
if uint16(len(z.File)) != uint16(end.directoryRecords) { // only compare 16 bits here
|
||||
// Return the readDirectoryHeader error if we read
|
||||
// the wrong number of directory entries.
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RegisterDecompressor registers or overrides a custom decompressor for a
|
||||
// specific method ID. If a decompressor for a given method is not found,
|
||||
// Reader will default to looking up the decompressor at the package level.
|
||||
func (z *Reader) RegisterDecompressor(method uint16, dcomp Decompressor) {
|
||||
if z.decompressors == nil {
|
||||
z.decompressors = make(map[uint16]Decompressor)
|
||||
}
|
||||
z.decompressors[method] = dcomp
|
||||
}
|
||||
|
||||
func (z *Reader) decompressor(method uint16) Decompressor {
|
||||
dcomp := z.decompressors[method]
|
||||
if dcomp == nil {
|
||||
dcomp = decompressor(method)
|
||||
}
|
||||
return dcomp
|
||||
}
|
||||
|
||||
// Close closes the Zip file, rendering it unusable for I/O.
|
||||
func (rc *ReadCloser) Close() error {
|
||||
return rc.f.Close()
|
||||
}
|
||||
|
||||
// DataOffset returns the offset of the file's possibly-compressed
|
||||
// data, relative to the beginning of the zip file.
|
||||
//
|
||||
// Most callers should instead use Open, which transparently
|
||||
// decompresses data and verifies checksums.
|
||||
func (f *File) DataOffset() (offset int64, err error) {
|
||||
bodyOffset, err := f.findBodyOffset()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
return f.headerOffset + bodyOffset, nil
|
||||
}
|
||||
|
||||
// Open returns a ReadCloser that provides access to the File's contents.
|
||||
// Multiple files may be read concurrently.
|
||||
func (f *File) Open() (io.ReadCloser, error) {
|
||||
bodyOffset, err := f.findBodyOffset()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
size := int64(f.CompressedSize64)
|
||||
r := io.NewSectionReader(f.zipr, f.headerOffset+bodyOffset, size)
|
||||
dcomp := f.zip.decompressor(f.Method)
|
||||
if dcomp == nil {
|
||||
return nil, ErrAlgorithm
|
||||
}
|
||||
var rc io.ReadCloser = dcomp(r)
|
||||
rc = &checksumReader{
|
||||
rc: rc,
|
||||
hash: crc32.NewIEEE(),
|
||||
f: f,
|
||||
}
|
||||
return rc, nil
|
||||
}
|
||||
|
||||
// OpenRaw returns a Reader that provides access to the File's contents without
|
||||
// decompression.
|
||||
func (f *File) OpenRaw() (io.Reader, error) {
|
||||
bodyOffset, err := f.findBodyOffset()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r := io.NewSectionReader(f.zipr, f.headerOffset+bodyOffset, int64(f.CompressedSize64))
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func (f *File) readDataDescriptor() {
|
||||
if !f.hasDataDescriptor() {
|
||||
return
|
||||
}
|
||||
|
||||
bodyOffset, err := f.findBodyOffset()
|
||||
if err != nil {
|
||||
f.descErr = err
|
||||
return
|
||||
}
|
||||
|
||||
// In section 4.3.9.2 of the spec: "However ZIP64 format MAY be used
|
||||
// regardless of the size of a file. When extracting, if the zip64
|
||||
// extended information extra field is present for the file the
|
||||
// compressed and uncompressed sizes will be 8 byte values."
|
||||
//
|
||||
// Historically, this package has used the compressed and uncompressed
|
||||
// sizes from the central directory to determine if the package is
|
||||
// zip64.
|
||||
//
|
||||
// For this case we allow either the extra field or sizes to determine
|
||||
// the data descriptor length.
|
||||
zip64 := f.zip64 || f.isZip64()
|
||||
n := int64(dataDescriptorLen)
|
||||
if zip64 {
|
||||
n = dataDescriptor64Len
|
||||
}
|
||||
size := int64(f.CompressedSize64)
|
||||
r := io.NewSectionReader(f.zipr, f.headerOffset+bodyOffset+size, n)
|
||||
dd, err := readDataDescriptor(r, zip64)
|
||||
if err != nil {
|
||||
f.descErr = err
|
||||
return
|
||||
}
|
||||
f.CRC32 = dd.crc32
|
||||
}
|
||||
|
||||
type checksumReader struct {
|
||||
rc io.ReadCloser
|
||||
hash hash.Hash32
|
||||
nread uint64 // number of bytes read so far
|
||||
f *File
|
||||
err error // sticky error
|
||||
}
|
||||
|
||||
func (r *checksumReader) Stat() (fs.FileInfo, error) {
|
||||
return headerFileInfo{&r.f.FileHeader}, nil
|
||||
}
|
||||
|
||||
func (r *checksumReader) Read(b []byte) (n int, err error) {
|
||||
if r.err != nil {
|
||||
return 0, r.err
|
||||
}
|
||||
n, err = r.rc.Read(b)
|
||||
r.hash.Write(b[:n])
|
||||
r.nread += uint64(n)
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
if err == io.EOF {
|
||||
if r.nread != r.f.UncompressedSize64 {
|
||||
return 0, io.ErrUnexpectedEOF
|
||||
}
|
||||
if r.f.hasDataDescriptor() {
|
||||
if r.f.descErr != nil {
|
||||
if r.f.descErr == io.EOF {
|
||||
err = io.ErrUnexpectedEOF
|
||||
} else {
|
||||
err = r.f.descErr
|
||||
}
|
||||
} else if r.hash.Sum32() != r.f.CRC32 {
|
||||
err = ErrChecksum
|
||||
}
|
||||
} else {
|
||||
// If there's not a data descriptor, we still compare
|
||||
// the CRC32 of what we've read against the file header
|
||||
// or TOC's CRC32, if it seems like it was set.
|
||||
if r.f.CRC32 != 0 && r.hash.Sum32() != r.f.CRC32 {
|
||||
err = ErrChecksum
|
||||
}
|
||||
}
|
||||
}
|
||||
r.err = err
|
||||
return
|
||||
}
|
||||
|
||||
func (r *checksumReader) Close() error { return r.rc.Close() }
|
||||
|
||||
// findBodyOffset does the minimum work to verify the file has a header
|
||||
// and returns the file body offset.
|
||||
func (f *File) findBodyOffset() (int64, error) {
|
||||
var buf [fileHeaderLen]byte
|
||||
if _, err := f.zipr.ReadAt(buf[:], f.headerOffset); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
b := readBuf(buf[:])
|
||||
if sig := b.uint32(); sig != fileHeaderSignature {
|
||||
return 0, ErrFormat
|
||||
}
|
||||
b = b[22:] // skip over most of the header
|
||||
filenameLen := int(b.uint16())
|
||||
extraLen := int(b.uint16())
|
||||
return int64(fileHeaderLen + filenameLen + extraLen), nil
|
||||
}
|
||||
|
||||
// readDirectoryHeader attempts to read a directory header from r.
|
||||
// It returns io.ErrUnexpectedEOF if it cannot read a complete header,
|
||||
// and ErrFormat if it doesn't find a valid header signature.
|
||||
func readDirectoryHeader(f *File, r io.Reader) error {
|
||||
var buf [directoryHeaderLen]byte
|
||||
if _, err := io.ReadFull(r, buf[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
b := readBuf(buf[:])
|
||||
if sig := b.uint32(); sig != directoryHeaderSignature {
|
||||
return ErrFormat
|
||||
}
|
||||
f.CreatorVersion = b.uint16()
|
||||
f.ReaderVersion = b.uint16()
|
||||
f.Flags = b.uint16()
|
||||
f.Method = b.uint16()
|
||||
f.ModifiedTime = b.uint16()
|
||||
f.ModifiedDate = b.uint16()
|
||||
f.CRC32 = b.uint32()
|
||||
f.CompressedSize = b.uint32()
|
||||
f.UncompressedSize = b.uint32()
|
||||
f.CompressedSize64 = uint64(f.CompressedSize)
|
||||
f.UncompressedSize64 = uint64(f.UncompressedSize)
|
||||
filenameLen := int(b.uint16())
|
||||
extraLen := int(b.uint16())
|
||||
commentLen := int(b.uint16())
|
||||
b = b[4:] // skipped start disk number and internal attributes (2x uint16)
|
||||
f.ExternalAttrs = b.uint32()
|
||||
f.headerOffset = int64(b.uint32())
|
||||
d := make([]byte, filenameLen+extraLen+commentLen)
|
||||
if _, err := io.ReadFull(r, d); err != nil {
|
||||
return err
|
||||
}
|
||||
f.Name = string(d[:filenameLen])
|
||||
f.Extra = d[filenameLen : filenameLen+extraLen]
|
||||
f.Comment = string(d[filenameLen+extraLen:])
|
||||
|
||||
// Determine the character encoding.
|
||||
utf8Valid1, utf8Require1 := detectUTF8(f.Name)
|
||||
utf8Valid2, utf8Require2 := detectUTF8(f.Comment)
|
||||
switch {
|
||||
case !utf8Valid1 || !utf8Valid2:
|
||||
// Name and Comment definitely not UTF-8.
|
||||
f.NonUTF8 = true
|
||||
case !utf8Require1 && !utf8Require2:
|
||||
// Name and Comment use only single-byte runes that overlap with UTF-8.
|
||||
f.NonUTF8 = false
|
||||
default:
|
||||
// Might be UTF-8, might be some other encoding; preserve existing flag.
|
||||
// Some ZIP writers use UTF-8 encoding without setting the UTF-8 flag.
|
||||
// Since it is impossible to always distinguish valid UTF-8 from some
|
||||
// other encoding (e.g., GBK or Shift-JIS), we trust the flag.
|
||||
f.NonUTF8 = f.Flags&0x800 == 0
|
||||
}
|
||||
|
||||
needUSize := f.UncompressedSize == ^uint32(0)
|
||||
needCSize := f.CompressedSize == ^uint32(0)
|
||||
needHeaderOffset := f.headerOffset == int64(^uint32(0))
|
||||
|
||||
// Best effort to find what we need.
|
||||
// Other zip authors might not even follow the basic format,
|
||||
// and we'll just ignore the Extra content in that case.
|
||||
var modified time.Time
|
||||
parseExtras:
|
||||
for extra := readBuf(f.Extra); len(extra) >= 4; { // need at least tag and size
|
||||
fieldTag := extra.uint16()
|
||||
fieldSize := int(extra.uint16())
|
||||
if len(extra) < fieldSize {
|
||||
break
|
||||
}
|
||||
fieldBuf := extra.sub(fieldSize)
|
||||
|
||||
switch fieldTag {
|
||||
case zip64ExtraID:
|
||||
f.zip64 = true
|
||||
|
||||
// update directory values from the zip64 extra block.
|
||||
// They should only be consulted if the sizes read earlier
|
||||
// are maxed out.
|
||||
// See golang.org/issue/13367.
|
||||
if needUSize {
|
||||
needUSize = false
|
||||
if len(fieldBuf) < 8 {
|
||||
return ErrFormat
|
||||
}
|
||||
f.UncompressedSize64 = fieldBuf.uint64()
|
||||
}
|
||||
if needCSize {
|
||||
needCSize = false
|
||||
if len(fieldBuf) < 8 {
|
||||
return ErrFormat
|
||||
}
|
||||
f.CompressedSize64 = fieldBuf.uint64()
|
||||
}
|
||||
if needHeaderOffset {
|
||||
needHeaderOffset = false
|
||||
if len(fieldBuf) < 8 {
|
||||
return ErrFormat
|
||||
}
|
||||
f.headerOffset = int64(fieldBuf.uint64())
|
||||
}
|
||||
case ntfsExtraID:
|
||||
if len(fieldBuf) < 4 {
|
||||
continue parseExtras
|
||||
}
|
||||
fieldBuf.uint32() // reserved (ignored)
|
||||
for len(fieldBuf) >= 4 { // need at least tag and size
|
||||
attrTag := fieldBuf.uint16()
|
||||
attrSize := int(fieldBuf.uint16())
|
||||
if len(fieldBuf) < attrSize {
|
||||
continue parseExtras
|
||||
}
|
||||
attrBuf := fieldBuf.sub(attrSize)
|
||||
if attrTag != 1 || attrSize != 24 {
|
||||
continue // Ignore irrelevant attributes
|
||||
}
|
||||
|
||||
const ticksPerSecond = 1e7 // Windows timestamp resolution
|
||||
ts := int64(attrBuf.uint64()) // ModTime since Windows epoch
|
||||
secs := int64(ts / ticksPerSecond)
|
||||
nsecs := (1e9 / ticksPerSecond) * int64(ts%ticksPerSecond)
|
||||
epoch := time.Date(1601, time.January, 1, 0, 0, 0, 0, time.UTC)
|
||||
modified = time.Unix(epoch.Unix()+secs, nsecs)
|
||||
}
|
||||
case unixExtraID, infoZipUnixExtraID:
|
||||
if len(fieldBuf) < 8 {
|
||||
continue parseExtras
|
||||
}
|
||||
fieldBuf.uint32() // AcTime (ignored)
|
||||
ts := int64(fieldBuf.uint32()) // ModTime since Unix epoch
|
||||
modified = time.Unix(ts, 0)
|
||||
case extTimeExtraID:
|
||||
if len(fieldBuf) < 5 || fieldBuf.uint8()&1 == 0 {
|
||||
continue parseExtras
|
||||
}
|
||||
ts := int64(fieldBuf.uint32()) // ModTime since Unix epoch
|
||||
modified = time.Unix(ts, 0)
|
||||
}
|
||||
}
|
||||
|
||||
msdosModified := msDosTimeToTime(f.ModifiedDate, f.ModifiedTime)
|
||||
f.Modified = msdosModified
|
||||
if !modified.IsZero() {
|
||||
f.Modified = modified.UTC()
|
||||
|
||||
// If legacy MS-DOS timestamps are set, we can use the delta between
|
||||
// the legacy and extended versions to estimate timezone offset.
|
||||
//
|
||||
// A non-UTC timezone is always used (even if offset is zero).
|
||||
// Thus, FileHeader.Modified.Location() == time.UTC is useful for
|
||||
// determining whether extended timestamps are present.
|
||||
// This is necessary for users that need to do additional time
|
||||
// calculations when dealing with legacy ZIP formats.
|
||||
if f.ModifiedTime != 0 || f.ModifiedDate != 0 {
|
||||
f.Modified = modified.In(timeZone(msdosModified.Sub(modified)))
|
||||
}
|
||||
}
|
||||
|
||||
// Assume that uncompressed size 2³²-1 could plausibly happen in
|
||||
// an old zip32 file that was sharding inputs into the largest chunks
|
||||
// possible (or is just malicious; search the web for 42.zip).
|
||||
// If needUSize is true still, it means we didn't see a zip64 extension.
|
||||
// As long as the compressed size is not also 2³²-1 (implausible)
|
||||
// and the header is not also 2³²-1 (equally implausible),
|
||||
// accept the uncompressed size 2³²-1 as valid.
|
||||
// If nothing else, this keeps archive/zip working with 42.zip.
|
||||
_ = needUSize
|
||||
|
||||
if needCSize || needHeaderOffset {
|
||||
return ErrFormat
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func readDataDescriptor(r io.Reader, zip64 bool) (*dataDescriptor, error) {
|
||||
// Create enough space for the largest possible size
|
||||
var buf [dataDescriptor64Len]byte
|
||||
|
||||
// The spec says: "Although not originally assigned a
|
||||
// signature, the value 0x08074b50 has commonly been adopted
|
||||
// as a signature value for the data descriptor record.
|
||||
// Implementers should be aware that ZIP files may be
|
||||
// encountered with or without this signature marking data
|
||||
// descriptors and should account for either case when reading
|
||||
// ZIP files to ensure compatibility."
|
||||
//
|
||||
// First read just those 4 bytes to see if the signature exists.
|
||||
if _, err := io.ReadFull(r, buf[:4]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
off := 0
|
||||
maybeSig := readBuf(buf[:4])
|
||||
if maybeSig.uint32() != dataDescriptorSignature {
|
||||
// No data descriptor signature. Keep these four
|
||||
// bytes.
|
||||
off += 4
|
||||
}
|
||||
|
||||
end := dataDescriptorLen - 4
|
||||
if zip64 {
|
||||
end = dataDescriptor64Len - 4
|
||||
}
|
||||
if _, err := io.ReadFull(r, buf[off:end]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
b := readBuf(buf[:end])
|
||||
|
||||
out := &dataDescriptor{
|
||||
crc32: b.uint32(),
|
||||
}
|
||||
|
||||
if zip64 {
|
||||
out.compressedSize = b.uint64()
|
||||
out.uncompressedSize = b.uint64()
|
||||
} else {
|
||||
out.compressedSize = uint64(b.uint32())
|
||||
out.uncompressedSize = uint64(b.uint32())
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func readDirectoryEnd(r io.ReaderAt, size int64) (dir *directoryEnd, err error) {
|
||||
// look for directoryEndSignature in the last 1k, then in the last 65k
|
||||
var buf []byte
|
||||
var directoryEndOffset int64
|
||||
for i, bLen := range []int64{1024, 65 * 1024} {
|
||||
if bLen > size {
|
||||
bLen = size
|
||||
}
|
||||
buf = make([]byte, int(bLen))
|
||||
if _, err := r.ReadAt(buf, size-bLen); err != nil && err != io.EOF {
|
||||
return nil, err
|
||||
}
|
||||
if p := findSignatureInBlock(buf); p >= 0 {
|
||||
buf = buf[p:]
|
||||
directoryEndOffset = size - bLen + int64(p)
|
||||
break
|
||||
}
|
||||
if i == 1 || bLen == size {
|
||||
return nil, ErrFormat
|
||||
}
|
||||
}
|
||||
|
||||
// read header into struct
|
||||
b := readBuf(buf[4:]) // skip signature
|
||||
d := &directoryEnd{
|
||||
diskNbr: uint32(b.uint16()),
|
||||
dirDiskNbr: uint32(b.uint16()),
|
||||
dirRecordsThisDisk: uint64(b.uint16()),
|
||||
directoryRecords: uint64(b.uint16()),
|
||||
directorySize: uint64(b.uint32()),
|
||||
directoryOffset: uint64(b.uint32()),
|
||||
commentLen: b.uint16(),
|
||||
}
|
||||
l := int(d.commentLen)
|
||||
if l > len(b) {
|
||||
return nil, errors.New("zip: invalid comment length")
|
||||
}
|
||||
d.comment = string(b[:l])
|
||||
|
||||
// These values mean that the file can be a zip64 file
|
||||
if d.directoryRecords == 0xffff || d.directorySize == 0xffff || d.directoryOffset == 0xffffffff {
|
||||
p, err := findDirectory64End(r, directoryEndOffset)
|
||||
if err == nil && p >= 0 {
|
||||
err = readDirectory64End(r, p, d)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
// Make sure directoryOffset points to somewhere in our file.
|
||||
if o := int64(d.directoryOffset); o < 0 || o >= size {
|
||||
return nil, ErrFormat
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// findDirectory64End tries to read the zip64 locator just before the
|
||||
// directory end and returns the offset of the zip64 directory end if
|
||||
// found.
|
||||
func findDirectory64End(r io.ReaderAt, directoryEndOffset int64) (int64, error) {
|
||||
locOffset := directoryEndOffset - directory64LocLen
|
||||
if locOffset < 0 {
|
||||
return -1, nil // no need to look for a header outside the file
|
||||
}
|
||||
buf := make([]byte, directory64LocLen)
|
||||
if _, err := r.ReadAt(buf, locOffset); err != nil {
|
||||
return -1, err
|
||||
}
|
||||
b := readBuf(buf)
|
||||
if sig := b.uint32(); sig != directory64LocSignature {
|
||||
return -1, nil
|
||||
}
|
||||
if b.uint32() != 0 { // number of the disk with the start of the zip64 end of central directory
|
||||
return -1, nil // the file is not a valid zip64-file
|
||||
}
|
||||
p := b.uint64() // relative offset of the zip64 end of central directory record
|
||||
if b.uint32() != 1 { // total number of disks
|
||||
return -1, nil // the file is not a valid zip64-file
|
||||
}
|
||||
return int64(p), nil
|
||||
}
|
||||
|
||||
// readDirectory64End reads the zip64 directory end and updates the
|
||||
// directory end with the zip64 directory end values.
|
||||
func readDirectory64End(r io.ReaderAt, offset int64, d *directoryEnd) (err error) {
|
||||
buf := make([]byte, directory64EndLen)
|
||||
if _, err := r.ReadAt(buf, offset); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
b := readBuf(buf)
|
||||
if sig := b.uint32(); sig != directory64EndSignature {
|
||||
return ErrFormat
|
||||
}
|
||||
|
||||
b = b[12:] // skip dir size, version and version needed (uint64 + 2x uint16)
|
||||
d.diskNbr = b.uint32() // number of this disk
|
||||
d.dirDiskNbr = b.uint32() // number of the disk with the start of the central directory
|
||||
d.dirRecordsThisDisk = b.uint64() // total number of entries in the central directory on this disk
|
||||
d.directoryRecords = b.uint64() // total number of entries in the central directory
|
||||
d.directorySize = b.uint64() // size of the central directory
|
||||
d.directoryOffset = b.uint64() // offset of start of central directory with respect to the starting disk number
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func findSignatureInBlock(b []byte) int {
|
||||
for i := len(b) - directoryEndLen; i >= 0; i-- {
|
||||
// defined from directoryEndSignature in struct.go
|
||||
if b[i] == 'P' && b[i+1] == 'K' && b[i+2] == 0x05 && b[i+3] == 0x06 {
|
||||
// n is length of comment
|
||||
n := int(b[i+directoryEndLen-2]) | int(b[i+directoryEndLen-1])<<8
|
||||
if n+directoryEndLen+i <= len(b) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
type readBuf []byte
|
||||
|
||||
func (b *readBuf) uint8() uint8 {
|
||||
v := (*b)[0]
|
||||
*b = (*b)[1:]
|
||||
return v
|
||||
}
|
||||
|
||||
func (b *readBuf) uint16() uint16 {
|
||||
v := binary.LittleEndian.Uint16(*b)
|
||||
*b = (*b)[2:]
|
||||
return v
|
||||
}
|
||||
|
||||
func (b *readBuf) uint32() uint32 {
|
||||
v := binary.LittleEndian.Uint32(*b)
|
||||
*b = (*b)[4:]
|
||||
return v
|
||||
}
|
||||
|
||||
func (b *readBuf) uint64() uint64 {
|
||||
v := binary.LittleEndian.Uint64(*b)
|
||||
*b = (*b)[8:]
|
||||
return v
|
||||
}
|
||||
|
||||
func (b *readBuf) sub(n int) readBuf {
|
||||
b2 := (*b)[:n]
|
||||
*b = (*b)[n:]
|
||||
return b2
|
||||
}
|
||||
|
||||
// A fileListEntry is a File and its ename.
|
||||
// If file == nil, the fileListEntry describes a directory without metadata.
|
||||
type fileListEntry struct {
|
||||
name string
|
||||
file *File
|
||||
isDir bool
|
||||
}
|
||||
|
||||
type fileInfoDirEntry interface {
|
||||
fs.FileInfo
|
||||
fs.DirEntry
|
||||
}
|
||||
|
||||
func (e *fileListEntry) stat() fileInfoDirEntry {
|
||||
if !e.isDir {
|
||||
return headerFileInfo{&e.file.FileHeader}
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
// Only used for directories.
|
||||
func (f *fileListEntry) Name() string { _, elem, _ := split(f.name); return elem }
|
||||
func (f *fileListEntry) Size() int64 { return 0 }
|
||||
func (f *fileListEntry) Mode() fs.FileMode { return fs.ModeDir | 0555 }
|
||||
func (f *fileListEntry) Type() fs.FileMode { return fs.ModeDir }
|
||||
func (f *fileListEntry) IsDir() bool { return true }
|
||||
func (f *fileListEntry) Sys() interface{} { return nil }
|
||||
|
||||
func (f *fileListEntry) ModTime() time.Time {
|
||||
if f.file == nil {
|
||||
return time.Time{}
|
||||
}
|
||||
return f.file.FileHeader.Modified.UTC()
|
||||
}
|
||||
|
||||
func (f *fileListEntry) Info() (fs.FileInfo, error) { return f, nil }
|
||||
|
||||
// toValidName coerces name to be a valid name for fs.FS.Open.
|
||||
func toValidName(name string) string {
|
||||
name = strings.ReplaceAll(name, `\`, `/`)
|
||||
p := path.Clean(name)
|
||||
if strings.HasPrefix(p, "/") {
|
||||
p = p[len("/"):]
|
||||
}
|
||||
for strings.HasPrefix(p, "../") {
|
||||
p = p[len("../"):]
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func (r *Reader) initFileList() {
|
||||
r.fileListOnce.Do(func() {
|
||||
dirs := make(map[string]bool)
|
||||
knownDirs := make(map[string]bool)
|
||||
for _, file := range r.File {
|
||||
isDir := len(file.Name) > 0 && file.Name[len(file.Name)-1] == '/'
|
||||
name := toValidName(file.Name)
|
||||
for dir := path.Dir(name); dir != "."; dir = path.Dir(dir) {
|
||||
dirs[dir] = true
|
||||
}
|
||||
entry := fileListEntry{
|
||||
name: name,
|
||||
file: file,
|
||||
isDir: isDir,
|
||||
}
|
||||
r.fileList = append(r.fileList, entry)
|
||||
if isDir {
|
||||
knownDirs[name] = true
|
||||
}
|
||||
}
|
||||
for dir := range dirs {
|
||||
if !knownDirs[dir] {
|
||||
entry := fileListEntry{
|
||||
name: dir,
|
||||
file: nil,
|
||||
isDir: true,
|
||||
}
|
||||
r.fileList = append(r.fileList, entry)
|
||||
}
|
||||
}
|
||||
|
||||
sort.Slice(r.fileList, func(i, j int) bool { return fileEntryLess(r.fileList[i].name, r.fileList[j].name) })
|
||||
})
|
||||
}
|
||||
|
||||
func fileEntryLess(x, y string) bool {
|
||||
xdir, xelem, _ := split(x)
|
||||
ydir, yelem, _ := split(y)
|
||||
return xdir < ydir || xdir == ydir && xelem < yelem
|
||||
}
|
||||
|
||||
// Open opens the named file in the ZIP archive,
|
||||
// using the semantics of fs.FS.Open:
|
||||
// paths are always slash separated, with no
|
||||
// leading / or ../ elements.
|
||||
func (r *Reader) Open(name string) (fs.File, error) {
|
||||
if !fs.ValidPath(name) {
|
||||
return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrInvalid}
|
||||
}
|
||||
|
||||
r.initFileList()
|
||||
|
||||
e := r.openLookup(name)
|
||||
if e == nil {
|
||||
return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist}
|
||||
}
|
||||
if e.isDir {
|
||||
return &openDir{e, r.openReadDir(name), 0}, nil
|
||||
}
|
||||
rc, err := e.file.Open()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return rc.(fs.File), nil
|
||||
}
|
||||
|
||||
func split(name string) (dir, elem string, isDir bool) {
|
||||
if name[len(name)-1] == '/' {
|
||||
isDir = true
|
||||
name = name[:len(name)-1]
|
||||
}
|
||||
i := len(name) - 1
|
||||
for i >= 0 && name[i] != '/' {
|
||||
i--
|
||||
}
|
||||
if i < 0 {
|
||||
return ".", name, isDir
|
||||
}
|
||||
return name[:i], name[i+1:], isDir
|
||||
}
|
||||
|
||||
var dotFile = &fileListEntry{name: "./", isDir: true}
|
||||
|
||||
func (r *Reader) openLookup(name string) *fileListEntry {
|
||||
if name == "." {
|
||||
return dotFile
|
||||
}
|
||||
|
||||
dir, elem, _ := split(name)
|
||||
files := r.fileList
|
||||
i := sort.Search(len(files), func(i int) bool {
|
||||
idir, ielem, _ := split(files[i].name)
|
||||
return idir > dir || idir == dir && ielem >= elem
|
||||
})
|
||||
if i < len(files) {
|
||||
fname := files[i].name
|
||||
if fname == name || len(fname) == len(name)+1 && fname[len(name)] == '/' && fname[:len(name)] == name {
|
||||
return &files[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Reader) openReadDir(dir string) []fileListEntry {
|
||||
files := r.fileList
|
||||
i := sort.Search(len(files), func(i int) bool {
|
||||
idir, _, _ := split(files[i].name)
|
||||
return idir >= dir
|
||||
})
|
||||
j := sort.Search(len(files), func(j int) bool {
|
||||
jdir, _, _ := split(files[j].name)
|
||||
return jdir > dir
|
||||
})
|
||||
return files[i:j]
|
||||
}
|
||||
|
||||
type openDir struct {
|
||||
e *fileListEntry
|
||||
files []fileListEntry
|
||||
offset int
|
||||
}
|
||||
|
||||
func (d *openDir) Close() error { return nil }
|
||||
func (d *openDir) Stat() (fs.FileInfo, error) { return d.e.stat(), nil }
|
||||
|
||||
func (d *openDir) Read([]byte) (int, error) {
|
||||
return 0, &fs.PathError{Op: "read", Path: d.e.name, Err: errors.New("is a directory")}
|
||||
}
|
||||
|
||||
func (d *openDir) ReadDir(count int) ([]fs.DirEntry, error) {
|
||||
n := len(d.files) - d.offset
|
||||
if count > 0 && n > count {
|
||||
n = count
|
||||
}
|
||||
if n == 0 {
|
||||
if count <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, io.EOF
|
||||
}
|
||||
list := make([]fs.DirEntry, n)
|
||||
for i := range list {
|
||||
list[i] = d.files[d.offset+i].stat()
|
||||
}
|
||||
d.offset += n
|
||||
return list, nil
|
||||
}
|
||||
626
vendor/github.com/klauspost/compress/zip/reader_leg.go
сгенерированный
поставляемый
Обычный файл
626
vendor/github.com/klauspost/compress/zip/reader_leg.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,626 @@
|
||||
// Copyright 2010 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
//go:build !go1.16
|
||||
// +build !go1.16
|
||||
|
||||
package zip
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrFormat = errors.New("zip: not a valid zip file")
|
||||
ErrAlgorithm = errors.New("zip: unsupported compression algorithm")
|
||||
ErrChecksum = errors.New("zip: checksum error")
|
||||
)
|
||||
|
||||
type Reader struct {
|
||||
r io.ReaderAt
|
||||
File []*File
|
||||
Comment string
|
||||
decompressors map[uint16]Decompressor
|
||||
}
|
||||
|
||||
type ReadCloser struct {
|
||||
f *os.File
|
||||
Reader
|
||||
}
|
||||
|
||||
type File struct {
|
||||
FileHeader
|
||||
zip *Reader
|
||||
zipr io.ReaderAt
|
||||
zipsize int64
|
||||
headerOffset int64
|
||||
}
|
||||
|
||||
func (f *File) hasDataDescriptor() bool {
|
||||
return f.Flags&0x8 != 0
|
||||
}
|
||||
|
||||
// OpenReader will open the Zip file specified by name and return a ReadCloser.
|
||||
func OpenReader(name string) (*ReadCloser, error) {
|
||||
f, err := os.Open(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fi, err := f.Stat()
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return nil, err
|
||||
}
|
||||
r := new(ReadCloser)
|
||||
if err := r.init(f, fi.Size()); err != nil {
|
||||
f.Close()
|
||||
return nil, err
|
||||
}
|
||||
r.f = f
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// NewReader returns a new Reader reading from r, which is assumed to
|
||||
// have the given size in bytes.
|
||||
func NewReader(r io.ReaderAt, size int64) (*Reader, error) {
|
||||
if size < 0 {
|
||||
return nil, errors.New("zip: size cannot be negative")
|
||||
}
|
||||
zr := new(Reader)
|
||||
if err := zr.init(r, size); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return zr, nil
|
||||
}
|
||||
|
||||
func (z *Reader) init(r io.ReaderAt, size int64) error {
|
||||
end, err := readDirectoryEnd(r, size)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if end.directoryRecords > uint64(size)/fileHeaderLen {
|
||||
return fmt.Errorf("archive/zip: TOC declares impossible %d files in %d byte zip", end.directoryRecords, size)
|
||||
}
|
||||
z.r = r
|
||||
z.File = make([]*File, 0, end.directoryRecords)
|
||||
z.Comment = end.comment
|
||||
rs := io.NewSectionReader(r, 0, size)
|
||||
if _, err = rs.Seek(int64(end.directoryOffset), io.SeekStart); err != nil {
|
||||
return err
|
||||
}
|
||||
buf := bufio.NewReader(rs)
|
||||
|
||||
// The count of files inside a zip is truncated to fit in a uint16.
|
||||
// Gloss over this by reading headers until we encounter
|
||||
// a bad one, and then only report an ErrFormat or UnexpectedEOF if
|
||||
// the file count modulo 65536 is incorrect.
|
||||
for {
|
||||
f := &File{zip: z, zipr: r, zipsize: size}
|
||||
err = readDirectoryHeader(f, buf)
|
||||
if err == ErrFormat || err == io.ErrUnexpectedEOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
z.File = append(z.File, f)
|
||||
}
|
||||
if uint16(len(z.File)) != uint16(end.directoryRecords) { // only compare 16 bits here
|
||||
// Return the readDirectoryHeader error if we read
|
||||
// the wrong number of directory entries.
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RegisterDecompressor registers or overrides a custom decompressor for a
|
||||
// specific method ID. If a decompressor for a given method is not found,
|
||||
// Reader will default to looking up the decompressor at the package level.
|
||||
func (z *Reader) RegisterDecompressor(method uint16, dcomp Decompressor) {
|
||||
if z.decompressors == nil {
|
||||
z.decompressors = make(map[uint16]Decompressor)
|
||||
}
|
||||
z.decompressors[method] = dcomp
|
||||
}
|
||||
|
||||
func (z *Reader) decompressor(method uint16) Decompressor {
|
||||
dcomp := z.decompressors[method]
|
||||
if dcomp == nil {
|
||||
dcomp = decompressor(method)
|
||||
}
|
||||
return dcomp
|
||||
}
|
||||
|
||||
// Close closes the Zip file, rendering it unusable for I/O.
|
||||
func (rc *ReadCloser) Close() error {
|
||||
return rc.f.Close()
|
||||
}
|
||||
|
||||
// DataOffset returns the offset of the file's possibly-compressed
|
||||
// data, relative to the beginning of the zip file.
|
||||
//
|
||||
// Most callers should instead use Open, which transparently
|
||||
// decompresses data and verifies checksums.
|
||||
func (f *File) DataOffset() (offset int64, err error) {
|
||||
bodyOffset, err := f.findBodyOffset()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
return f.headerOffset + bodyOffset, nil
|
||||
}
|
||||
|
||||
// Open returns a ReadCloser that provides access to the File's contents.
|
||||
// Multiple files may be read concurrently.
|
||||
func (f *File) Open() (io.ReadCloser, error) {
|
||||
bodyOffset, err := f.findBodyOffset()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
size := int64(f.CompressedSize64)
|
||||
r := io.NewSectionReader(f.zipr, f.headerOffset+bodyOffset, size)
|
||||
dcomp := f.zip.decompressor(f.Method)
|
||||
if dcomp == nil {
|
||||
return nil, ErrAlgorithm
|
||||
}
|
||||
var rc io.ReadCloser = dcomp(r)
|
||||
var desr io.Reader
|
||||
if f.hasDataDescriptor() {
|
||||
desr = io.NewSectionReader(f.zipr, f.headerOffset+bodyOffset+size, dataDescriptorLen)
|
||||
}
|
||||
rc = &checksumReader{
|
||||
rc: rc,
|
||||
hash: crc32.NewIEEE(),
|
||||
f: f,
|
||||
desr: desr,
|
||||
}
|
||||
return rc, nil
|
||||
}
|
||||
|
||||
// OpenRaw returns a Reader that returns the *compressed* output of the file.
|
||||
func (f *File) OpenRaw() (io.Reader, error) {
|
||||
bodyOffset, err := f.findBodyOffset()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
size := int64(f.CompressedSize64)
|
||||
return io.NewSectionReader(f.zipr, f.headerOffset+bodyOffset, size), nil
|
||||
}
|
||||
|
||||
type checksumReader struct {
|
||||
rc io.ReadCloser
|
||||
hash hash.Hash32
|
||||
nread uint64 // number of bytes read so far
|
||||
f *File
|
||||
desr io.Reader // if non-nil, where to read the data descriptor
|
||||
err error // sticky error
|
||||
}
|
||||
|
||||
func (r *checksumReader) Read(b []byte) (n int, err error) {
|
||||
if r.err != nil {
|
||||
return 0, r.err
|
||||
}
|
||||
n, err = r.rc.Read(b)
|
||||
r.hash.Write(b[:n])
|
||||
r.nread += uint64(n)
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
if err == io.EOF {
|
||||
if r.nread != r.f.UncompressedSize64 {
|
||||
return 0, io.ErrUnexpectedEOF
|
||||
}
|
||||
if r.desr != nil {
|
||||
if err1 := readDataDescriptor(r.desr, r.f); err1 != nil {
|
||||
if err1 == io.EOF {
|
||||
err = io.ErrUnexpectedEOF
|
||||
} else {
|
||||
err = err1
|
||||
}
|
||||
} else if r.hash.Sum32() != r.f.CRC32 {
|
||||
err = ErrChecksum
|
||||
}
|
||||
} else {
|
||||
// If there's not a data descriptor, we still compare
|
||||
// the CRC32 of what we've read against the file header
|
||||
// or TOC's CRC32, if it seems like it was set.
|
||||
if r.f.CRC32 != 0 && r.hash.Sum32() != r.f.CRC32 {
|
||||
err = ErrChecksum
|
||||
}
|
||||
}
|
||||
}
|
||||
r.err = err
|
||||
return
|
||||
}
|
||||
|
||||
func (r *checksumReader) Close() error { return r.rc.Close() }
|
||||
|
||||
// findBodyOffset does the minimum work to verify the file has a header
|
||||
// and returns the file body offset.
|
||||
func (f *File) findBodyOffset() (int64, error) {
|
||||
var buf [fileHeaderLen]byte
|
||||
if _, err := f.zipr.ReadAt(buf[:], f.headerOffset); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
b := readBuf(buf[:])
|
||||
if sig := b.uint32(); sig != fileHeaderSignature {
|
||||
return 0, ErrFormat
|
||||
}
|
||||
b = b[22:] // skip over most of the header
|
||||
filenameLen := int(b.uint16())
|
||||
extraLen := int(b.uint16())
|
||||
return int64(fileHeaderLen + filenameLen + extraLen), nil
|
||||
}
|
||||
|
||||
// readDirectoryHeader attempts to read a directory header from r.
|
||||
// It returns io.ErrUnexpectedEOF if it cannot read a complete header,
|
||||
// and ErrFormat if it doesn't find a valid header signature.
|
||||
func readDirectoryHeader(f *File, r io.Reader) error {
|
||||
var buf [directoryHeaderLen]byte
|
||||
if _, err := io.ReadFull(r, buf[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
b := readBuf(buf[:])
|
||||
if sig := b.uint32(); sig != directoryHeaderSignature {
|
||||
return ErrFormat
|
||||
}
|
||||
f.CreatorVersion = b.uint16()
|
||||
f.ReaderVersion = b.uint16()
|
||||
f.Flags = b.uint16()
|
||||
f.Method = b.uint16()
|
||||
f.ModifiedTime = b.uint16()
|
||||
f.ModifiedDate = b.uint16()
|
||||
f.CRC32 = b.uint32()
|
||||
f.CompressedSize = b.uint32()
|
||||
f.UncompressedSize = b.uint32()
|
||||
f.CompressedSize64 = uint64(f.CompressedSize)
|
||||
f.UncompressedSize64 = uint64(f.UncompressedSize)
|
||||
filenameLen := int(b.uint16())
|
||||
extraLen := int(b.uint16())
|
||||
commentLen := int(b.uint16())
|
||||
b = b[4:] // skipped start disk number and internal attributes (2x uint16)
|
||||
f.ExternalAttrs = b.uint32()
|
||||
f.headerOffset = int64(b.uint32())
|
||||
d := make([]byte, filenameLen+extraLen+commentLen)
|
||||
if _, err := io.ReadFull(r, d); err != nil {
|
||||
return err
|
||||
}
|
||||
f.Name = string(d[:filenameLen])
|
||||
f.Extra = d[filenameLen : filenameLen+extraLen]
|
||||
f.Comment = string(d[filenameLen+extraLen:])
|
||||
|
||||
// Determine the character encoding.
|
||||
utf8Valid1, utf8Require1 := detectUTF8(f.Name)
|
||||
utf8Valid2, utf8Require2 := detectUTF8(f.Comment)
|
||||
switch {
|
||||
case !utf8Valid1 || !utf8Valid2:
|
||||
// Name and Comment definitely not UTF-8.
|
||||
f.NonUTF8 = true
|
||||
case !utf8Require1 && !utf8Require2:
|
||||
// Name and Comment use only single-byte runes that overlap with UTF-8.
|
||||
f.NonUTF8 = false
|
||||
default:
|
||||
// Might be UTF-8, might be some other encoding; preserve existing flag.
|
||||
// Some ZIP writers use UTF-8 encoding without setting the UTF-8 flag.
|
||||
// Since it is impossible to always distinguish valid UTF-8 from some
|
||||
// other encoding (e.g., GBK or Shift-JIS), we trust the flag.
|
||||
f.NonUTF8 = f.Flags&0x800 == 0
|
||||
}
|
||||
|
||||
needUSize := f.UncompressedSize == ^uint32(0)
|
||||
needCSize := f.CompressedSize == ^uint32(0)
|
||||
needHeaderOffset := f.headerOffset == int64(^uint32(0))
|
||||
|
||||
// Best effort to find what we need.
|
||||
// Other zip authors might not even follow the basic format,
|
||||
// and we'll just ignore the Extra content in that case.
|
||||
var modified time.Time
|
||||
parseExtras:
|
||||
for extra := readBuf(f.Extra); len(extra) >= 4; { // need at least tag and size
|
||||
fieldTag := extra.uint16()
|
||||
fieldSize := int(extra.uint16())
|
||||
if len(extra) < fieldSize {
|
||||
break
|
||||
}
|
||||
fieldBuf := extra.sub(fieldSize)
|
||||
|
||||
switch fieldTag {
|
||||
case zip64ExtraID:
|
||||
// update directory values from the zip64 extra block.
|
||||
// They should only be consulted if the sizes read earlier
|
||||
// are maxed out.
|
||||
// See golang.org/issue/13367.
|
||||
if needUSize {
|
||||
needUSize = false
|
||||
if len(fieldBuf) < 8 {
|
||||
return ErrFormat
|
||||
}
|
||||
f.UncompressedSize64 = fieldBuf.uint64()
|
||||
}
|
||||
if needCSize {
|
||||
needCSize = false
|
||||
if len(fieldBuf) < 8 {
|
||||
return ErrFormat
|
||||
}
|
||||
f.CompressedSize64 = fieldBuf.uint64()
|
||||
}
|
||||
if needHeaderOffset {
|
||||
needHeaderOffset = false
|
||||
if len(fieldBuf) < 8 {
|
||||
return ErrFormat
|
||||
}
|
||||
f.headerOffset = int64(fieldBuf.uint64())
|
||||
}
|
||||
case ntfsExtraID:
|
||||
if len(fieldBuf) < 4 {
|
||||
continue parseExtras
|
||||
}
|
||||
fieldBuf.uint32() // reserved (ignored)
|
||||
for len(fieldBuf) >= 4 { // need at least tag and size
|
||||
attrTag := fieldBuf.uint16()
|
||||
attrSize := int(fieldBuf.uint16())
|
||||
if len(fieldBuf) < attrSize {
|
||||
continue parseExtras
|
||||
}
|
||||
attrBuf := fieldBuf.sub(attrSize)
|
||||
if attrTag != 1 || attrSize != 24 {
|
||||
continue // Ignore irrelevant attributes
|
||||
}
|
||||
|
||||
const ticksPerSecond = 1e7 // Windows timestamp resolution
|
||||
ts := int64(attrBuf.uint64()) // ModTime since Windows epoch
|
||||
secs := ts / ticksPerSecond
|
||||
nsecs := (1e9 / ticksPerSecond) * (ts % ticksPerSecond)
|
||||
epoch := time.Date(1601, time.January, 1, 0, 0, 0, 0, time.UTC)
|
||||
modified = time.Unix(epoch.Unix()+secs, nsecs)
|
||||
}
|
||||
case unixExtraID, infoZipUnixExtraID:
|
||||
if len(fieldBuf) < 8 {
|
||||
continue parseExtras
|
||||
}
|
||||
fieldBuf.uint32() // AcTime (ignored)
|
||||
ts := int64(fieldBuf.uint32()) // ModTime since Unix epoch
|
||||
modified = time.Unix(ts, 0)
|
||||
case extTimeExtraID:
|
||||
if len(fieldBuf) < 5 || fieldBuf.uint8()&1 == 0 {
|
||||
continue parseExtras
|
||||
}
|
||||
ts := int64(fieldBuf.uint32()) // ModTime since Unix epoch
|
||||
modified = time.Unix(ts, 0)
|
||||
}
|
||||
}
|
||||
|
||||
msdosModified := msDosTimeToTime(f.ModifiedDate, f.ModifiedTime)
|
||||
f.Modified = msdosModified
|
||||
if !modified.IsZero() {
|
||||
f.Modified = modified.UTC()
|
||||
|
||||
// If legacy MS-DOS timestamps are set, we can use the delta between
|
||||
// the legacy and extended versions to estimate timezone offset.
|
||||
//
|
||||
// A non-UTC timezone is always used (even if offset is zero).
|
||||
// Thus, FileHeader.Modified.Location() == time.UTC is useful for
|
||||
// determining whether extended timestamps are present.
|
||||
// This is necessary for users that need to do additional time
|
||||
// calculations when dealing with legacy ZIP formats.
|
||||
if f.ModifiedTime != 0 || f.ModifiedDate != 0 {
|
||||
f.Modified = modified.In(timeZone(msdosModified.Sub(modified)))
|
||||
}
|
||||
}
|
||||
|
||||
// Assume that uncompressed size 2³²-1 could plausibly happen in
|
||||
// an old zip32 file that was sharding inputs into the largest chunks
|
||||
// possible (or is just malicious; search the web for 42.zip).
|
||||
// If needUSize is true still, it means we didn't see a zip64 extension.
|
||||
// As long as the compressed size is not also 2³²-1 (implausible)
|
||||
// and the header is not also 2³²-1 (equally implausible),
|
||||
// accept the uncompressed size 2³²-1 as valid.
|
||||
// If nothing else, this keeps archive/zip working with 42.zip.
|
||||
_ = needUSize
|
||||
|
||||
if needCSize || needHeaderOffset {
|
||||
return ErrFormat
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func readDataDescriptor(r io.Reader, f *File) error {
|
||||
var buf [dataDescriptorLen]byte
|
||||
|
||||
// The spec says: "Although not originally assigned a
|
||||
// signature, the value 0x08074b50 has commonly been adopted
|
||||
// as a signature value for the data descriptor record.
|
||||
// Implementers should be aware that ZIP files may be
|
||||
// encountered with or without this signature marking data
|
||||
// descriptors and should account for either case when reading
|
||||
// ZIP files to ensure compatibility."
|
||||
//
|
||||
// dataDescriptorLen includes the size of the signature but
|
||||
// first read just those 4 bytes to see if it exists.
|
||||
if _, err := io.ReadFull(r, buf[:4]); err != nil {
|
||||
return err
|
||||
}
|
||||
off := 0
|
||||
maybeSig := readBuf(buf[:4])
|
||||
if maybeSig.uint32() != dataDescriptorSignature {
|
||||
// No data descriptor signature. Keep these four
|
||||
// bytes.
|
||||
off += 4
|
||||
}
|
||||
if _, err := io.ReadFull(r, buf[off:12]); err != nil {
|
||||
return err
|
||||
}
|
||||
b := readBuf(buf[:12])
|
||||
if b.uint32() != f.CRC32 {
|
||||
return ErrChecksum
|
||||
}
|
||||
|
||||
// The two sizes that follow here can be either 32 bits or 64 bits
|
||||
// but the spec is not very clear on this and different
|
||||
// interpretations has been made causing incompatibilities. We
|
||||
// already have the sizes from the central directory so we can
|
||||
// just ignore these.
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func readDirectoryEnd(r io.ReaderAt, size int64) (dir *directoryEnd, err error) {
|
||||
// look for directoryEndSignature in the last 1k, then in the last 65k
|
||||
var buf []byte
|
||||
var directoryEndOffset int64
|
||||
for i, bLen := range []int64{1024, 65 * 1024} {
|
||||
if bLen > size {
|
||||
bLen = size
|
||||
}
|
||||
buf = make([]byte, int(bLen))
|
||||
if _, err := r.ReadAt(buf, size-bLen); err != nil && err != io.EOF {
|
||||
return nil, err
|
||||
}
|
||||
if p := findSignatureInBlock(buf); p >= 0 {
|
||||
buf = buf[p:]
|
||||
directoryEndOffset = size - bLen + int64(p)
|
||||
break
|
||||
}
|
||||
if i == 1 || bLen == size {
|
||||
return nil, ErrFormat
|
||||
}
|
||||
}
|
||||
|
||||
// read header into struct
|
||||
b := readBuf(buf[4:]) // skip signature
|
||||
d := &directoryEnd{
|
||||
diskNbr: uint32(b.uint16()),
|
||||
dirDiskNbr: uint32(b.uint16()),
|
||||
dirRecordsThisDisk: uint64(b.uint16()),
|
||||
directoryRecords: uint64(b.uint16()),
|
||||
directorySize: uint64(b.uint32()),
|
||||
directoryOffset: uint64(b.uint32()),
|
||||
commentLen: b.uint16(),
|
||||
}
|
||||
l := int(d.commentLen)
|
||||
if l > len(b) {
|
||||
return nil, errors.New("zip: invalid comment length")
|
||||
}
|
||||
d.comment = string(b[:l])
|
||||
|
||||
// These values mean that the file can be a zip64 file
|
||||
if d.directoryRecords == 0xffff || d.directorySize == 0xffff || d.directoryOffset == 0xffffffff {
|
||||
p, err := findDirectory64End(r, directoryEndOffset)
|
||||
if err == nil && p >= 0 {
|
||||
err = readDirectory64End(r, p, d)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
// Make sure directoryOffset points to somewhere in our file.
|
||||
if o := int64(d.directoryOffset); o < 0 || o >= size {
|
||||
return nil, ErrFormat
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// findDirectory64End tries to read the zip64 locator just before the
|
||||
// directory end and returns the offset of the zip64 directory end if
|
||||
// found.
|
||||
func findDirectory64End(r io.ReaderAt, directoryEndOffset int64) (int64, error) {
|
||||
locOffset := directoryEndOffset - directory64LocLen
|
||||
if locOffset < 0 {
|
||||
return -1, nil // no need to look for a header outside the file
|
||||
}
|
||||
buf := make([]byte, directory64LocLen)
|
||||
if _, err := r.ReadAt(buf, locOffset); err != nil {
|
||||
return -1, err
|
||||
}
|
||||
b := readBuf(buf)
|
||||
if sig := b.uint32(); sig != directory64LocSignature {
|
||||
return -1, nil
|
||||
}
|
||||
if b.uint32() != 0 { // number of the disk with the start of the zip64 end of central directory
|
||||
return -1, nil // the file is not a valid zip64-file
|
||||
}
|
||||
p := b.uint64() // relative offset of the zip64 end of central directory record
|
||||
if b.uint32() != 1 { // total number of disks
|
||||
return -1, nil // the file is not a valid zip64-file
|
||||
}
|
||||
return int64(p), nil
|
||||
}
|
||||
|
||||
// readDirectory64End reads the zip64 directory end and updates the
|
||||
// directory end with the zip64 directory end values.
|
||||
func readDirectory64End(r io.ReaderAt, offset int64, d *directoryEnd) (err error) {
|
||||
buf := make([]byte, directory64EndLen)
|
||||
if _, err := r.ReadAt(buf, offset); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
b := readBuf(buf)
|
||||
if sig := b.uint32(); sig != directory64EndSignature {
|
||||
return ErrFormat
|
||||
}
|
||||
|
||||
b = b[12:] // skip dir size, version and version needed (uint64 + 2x uint16)
|
||||
d.diskNbr = b.uint32() // number of this disk
|
||||
d.dirDiskNbr = b.uint32() // number of the disk with the start of the central directory
|
||||
d.dirRecordsThisDisk = b.uint64() // total number of entries in the central directory on this disk
|
||||
d.directoryRecords = b.uint64() // total number of entries in the central directory
|
||||
d.directorySize = b.uint64() // size of the central directory
|
||||
d.directoryOffset = b.uint64() // offset of start of central directory with respect to the starting disk number
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func findSignatureInBlock(b []byte) int {
|
||||
for i := len(b) - directoryEndLen; i >= 0; i-- {
|
||||
// defined from directoryEndSignature in struct.go
|
||||
if b[i] == 'P' && b[i+1] == 'K' && b[i+2] == 0x05 && b[i+3] == 0x06 {
|
||||
// n is length of comment
|
||||
n := int(b[i+directoryEndLen-2]) | int(b[i+directoryEndLen-1])<<8
|
||||
if n+directoryEndLen+i <= len(b) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
type readBuf []byte
|
||||
|
||||
func (b *readBuf) uint8() uint8 {
|
||||
v := (*b)[0]
|
||||
*b = (*b)[1:]
|
||||
return v
|
||||
}
|
||||
|
||||
func (b *readBuf) uint16() uint16 {
|
||||
v := binary.LittleEndian.Uint16(*b)
|
||||
*b = (*b)[2:]
|
||||
return v
|
||||
}
|
||||
|
||||
func (b *readBuf) uint32() uint32 {
|
||||
v := binary.LittleEndian.Uint32(*b)
|
||||
*b = (*b)[4:]
|
||||
return v
|
||||
}
|
||||
|
||||
func (b *readBuf) uint64() uint64 {
|
||||
v := binary.LittleEndian.Uint64(*b)
|
||||
*b = (*b)[8:]
|
||||
return v
|
||||
}
|
||||
|
||||
func (b *readBuf) sub(n int) readBuf {
|
||||
b2 := (*b)[:n]
|
||||
*b = (*b)[n:]
|
||||
return b2
|
||||
}
|
||||
149
vendor/github.com/klauspost/compress/zip/register.go
сгенерированный
поставляемый
Обычный файл
149
vendor/github.com/klauspost/compress/zip/register.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,149 @@
|
||||
// Copyright 2010 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package zip
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"sync"
|
||||
|
||||
"github.com/klauspost/compress/flate"
|
||||
)
|
||||
|
||||
// A Compressor returns a new compressing writer, writing to w.
|
||||
// The WriteCloser's Close method must be used to flush pending data to w.
|
||||
// The Compressor itself must be safe to invoke from multiple goroutines
|
||||
// simultaneously, but each returned writer will be used only by
|
||||
// one goroutine at a time.
|
||||
type Compressor func(w io.Writer) (io.WriteCloser, error)
|
||||
|
||||
// A Decompressor returns a new decompressing reader, reading from r.
|
||||
// The ReadCloser's Close method must be used to release associated resources.
|
||||
// The Decompressor itself must be safe to invoke from multiple goroutines
|
||||
// simultaneously, but each returned reader will be used only by
|
||||
// one goroutine at a time.
|
||||
type Decompressor func(r io.Reader) io.ReadCloser
|
||||
|
||||
var flateWriterPool sync.Pool
|
||||
|
||||
func newFlateWriter(w io.Writer) io.WriteCloser {
|
||||
fw, ok := flateWriterPool.Get().(*flate.Writer)
|
||||
if ok {
|
||||
fw.Reset(w)
|
||||
} else {
|
||||
fw, _ = flate.NewWriter(w, 5)
|
||||
}
|
||||
return &pooledFlateWriter{fw: fw}
|
||||
}
|
||||
|
||||
type pooledFlateWriter struct {
|
||||
mu sync.Mutex // guards Close and Write
|
||||
fw *flate.Writer
|
||||
}
|
||||
|
||||
func (w *pooledFlateWriter) Write(p []byte) (n int, err error) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.fw == nil {
|
||||
return 0, errors.New("Write after Close")
|
||||
}
|
||||
return w.fw.Write(p)
|
||||
}
|
||||
|
||||
func (w *pooledFlateWriter) Close() error {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
var err error
|
||||
if w.fw != nil {
|
||||
err = w.fw.Close()
|
||||
flateWriterPool.Put(w.fw)
|
||||
w.fw = nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
var flateReaderPool sync.Pool
|
||||
|
||||
func newFlateReader(r io.Reader) io.ReadCloser {
|
||||
fr, ok := flateReaderPool.Get().(io.ReadCloser)
|
||||
if ok {
|
||||
fr.(flate.Resetter).Reset(r, nil)
|
||||
} else {
|
||||
fr = flate.NewReader(r)
|
||||
}
|
||||
return &pooledFlateReader{fr: fr}
|
||||
}
|
||||
|
||||
type pooledFlateReader struct {
|
||||
mu sync.Mutex // guards Close and Read
|
||||
fr io.ReadCloser
|
||||
}
|
||||
|
||||
func (r *pooledFlateReader) Read(p []byte) (n int, err error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.fr == nil {
|
||||
return 0, errors.New("Read after Close")
|
||||
}
|
||||
return r.fr.Read(p)
|
||||
}
|
||||
|
||||
func (r *pooledFlateReader) Close() error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
var err error
|
||||
if r.fr != nil {
|
||||
err = r.fr.Close()
|
||||
flateReaderPool.Put(r.fr)
|
||||
r.fr = nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
var (
|
||||
compressors sync.Map // map[uint16]Compressor
|
||||
decompressors sync.Map // map[uint16]Decompressor
|
||||
)
|
||||
|
||||
func init() {
|
||||
compressors.Store(Store, Compressor(func(w io.Writer) (io.WriteCloser, error) { return &nopCloser{w}, nil }))
|
||||
compressors.Store(Deflate, Compressor(func(w io.Writer) (io.WriteCloser, error) { return newFlateWriter(w), nil }))
|
||||
|
||||
decompressors.Store(Store, Decompressor(ioutil.NopCloser))
|
||||
decompressors.Store(Deflate, Decompressor(newFlateReader))
|
||||
}
|
||||
|
||||
// RegisterDecompressor allows custom decompressors for a specified method ID.
|
||||
// The common methods Store and Deflate are built in.
|
||||
func RegisterDecompressor(method uint16, dcomp Decompressor) {
|
||||
if _, dup := decompressors.LoadOrStore(method, dcomp); dup {
|
||||
panic("decompressor already registered")
|
||||
}
|
||||
}
|
||||
|
||||
// RegisterCompressor registers custom compressors for a specified method ID.
|
||||
// The common methods Store and Deflate are built in.
|
||||
func RegisterCompressor(method uint16, comp Compressor) {
|
||||
if _, dup := compressors.LoadOrStore(method, comp); dup {
|
||||
panic("compressor already registered")
|
||||
}
|
||||
}
|
||||
|
||||
func compressor(method uint16) Compressor {
|
||||
ci, ok := compressors.Load(method)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return ci.(Compressor)
|
||||
}
|
||||
|
||||
func decompressor(method uint16) Decompressor {
|
||||
di, ok := decompressors.Load(method)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return di.(Decompressor)
|
||||
}
|
||||
403
vendor/github.com/klauspost/compress/zip/struct.go
сгенерированный
поставляемый
Обычный файл
403
vendor/github.com/klauspost/compress/zip/struct.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,403 @@
|
||||
// Copyright 2010 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
//go:build go1.16
|
||||
// +build go1.16
|
||||
|
||||
/*
|
||||
Package zip provides support for reading and writing ZIP archives.
|
||||
|
||||
See: https://www.pkware.com/appnote
|
||||
|
||||
This package does not support disk spanning.
|
||||
|
||||
A note about ZIP64:
|
||||
|
||||
To be backwards compatible the FileHeader has both 32 and 64 bit Size
|
||||
fields. The 64 bit fields will always contain the correct value and
|
||||
for normal archives both fields will be the same. For files requiring
|
||||
the ZIP64 format the 32 bit fields will be 0xffffffff and the 64 bit
|
||||
fields must be used instead.
|
||||
*/
|
||||
package zip
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"path"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Compression methods.
|
||||
const (
|
||||
Store uint16 = 0 // no compression
|
||||
Deflate uint16 = 8 // DEFLATE compressed
|
||||
)
|
||||
|
||||
const (
|
||||
fileHeaderSignature = 0x04034b50
|
||||
directoryHeaderSignature = 0x02014b50
|
||||
directoryEndSignature = 0x06054b50
|
||||
directory64LocSignature = 0x07064b50
|
||||
directory64EndSignature = 0x06064b50
|
||||
dataDescriptorSignature = 0x08074b50 // de-facto standard; required by OS X Finder
|
||||
fileHeaderLen = 30 // + filename + extra
|
||||
directoryHeaderLen = 46 // + filename + extra + comment
|
||||
directoryEndLen = 22 // + comment
|
||||
dataDescriptorLen = 16 // four uint32: descriptor signature, crc32, compressed size, size
|
||||
dataDescriptor64Len = 24 // two uint32: signature, crc32 | two uint64: compressed size, size
|
||||
directory64LocLen = 20 //
|
||||
directory64EndLen = 56 // + extra
|
||||
|
||||
// Constants for the first byte in CreatorVersion.
|
||||
creatorFAT = 0
|
||||
creatorUnix = 3
|
||||
creatorNTFS = 11
|
||||
creatorVFAT = 14
|
||||
creatorMacOSX = 19
|
||||
|
||||
// Version numbers.
|
||||
zipVersion20 = 20 // 2.0
|
||||
zipVersion45 = 45 // 4.5 (reads and writes zip64 archives)
|
||||
|
||||
// Limits for non zip64 files.
|
||||
uint16max = (1 << 16) - 1
|
||||
uint32max = (1 << 32) - 1
|
||||
|
||||
// Extra header IDs.
|
||||
//
|
||||
// IDs 0..31 are reserved for official use by PKWARE.
|
||||
// IDs above that range are defined by third-party vendors.
|
||||
// Since ZIP lacked high precision timestamps (nor a official specification
|
||||
// of the timezone used for the date fields), many competing extra fields
|
||||
// have been invented. Pervasive use effectively makes them "official".
|
||||
//
|
||||
// See http://mdfs.net/Docs/Comp/Archiving/Zip/ExtraField
|
||||
zip64ExtraID = 0x0001 // Zip64 extended information
|
||||
ntfsExtraID = 0x000a // NTFS
|
||||
unixExtraID = 0x000d // UNIX
|
||||
extTimeExtraID = 0x5455 // Extended timestamp
|
||||
infoZipUnixExtraID = 0x5855 // Info-ZIP Unix extension
|
||||
)
|
||||
|
||||
// FileHeader describes a file within a zip file.
|
||||
// See the zip spec for details.
|
||||
type FileHeader struct {
|
||||
// Name is the name of the file.
|
||||
//
|
||||
// It must be a relative path, not start with a drive letter (such as "C:"),
|
||||
// and must use forward slashes instead of back slashes. A trailing slash
|
||||
// indicates that this file is a directory and should have no data.
|
||||
//
|
||||
// When reading zip files, the Name field is populated from
|
||||
// the zip file directly and is not validated for correctness.
|
||||
// It is the caller's responsibility to sanitize it as
|
||||
// appropriate, including canonicalizing slash directions,
|
||||
// validating that paths are relative, and preventing path
|
||||
// traversal through filenames ("../../../").
|
||||
Name string
|
||||
|
||||
// Comment is any arbitrary user-defined string shorter than 64KiB.
|
||||
Comment string
|
||||
|
||||
// NonUTF8 indicates that Name and Comment are not encoded in UTF-8.
|
||||
//
|
||||
// By specification, the only other encoding permitted should be CP-437,
|
||||
// but historically many ZIP readers interpret Name and Comment as whatever
|
||||
// the system's local character encoding happens to be.
|
||||
//
|
||||
// This flag should only be set if the user intends to encode a non-portable
|
||||
// ZIP file for a specific localized region. Otherwise, the Writer
|
||||
// automatically sets the ZIP format's UTF-8 flag for valid UTF-8 strings.
|
||||
NonUTF8 bool
|
||||
|
||||
CreatorVersion uint16
|
||||
ReaderVersion uint16
|
||||
Flags uint16
|
||||
|
||||
// Method is the compression method. If zero, Store is used.
|
||||
Method uint16
|
||||
|
||||
// Modified is the modified time of the file.
|
||||
//
|
||||
// When reading, an extended timestamp is preferred over the legacy MS-DOS
|
||||
// date field, and the offset between the times is used as the timezone.
|
||||
// If only the MS-DOS date is present, the timezone is assumed to be UTC.
|
||||
//
|
||||
// When writing, an extended timestamp (which is timezone-agnostic) is
|
||||
// always emitted. The legacy MS-DOS date field is encoded according to the
|
||||
// location of the Modified time.
|
||||
Modified time.Time
|
||||
ModifiedTime uint16 // Deprecated: Legacy MS-DOS date; use Modified instead.
|
||||
ModifiedDate uint16 // Deprecated: Legacy MS-DOS time; use Modified instead.
|
||||
|
||||
CRC32 uint32
|
||||
CompressedSize uint32 // Deprecated: Use CompressedSize64 instead.
|
||||
UncompressedSize uint32 // Deprecated: Use UncompressedSize64 instead.
|
||||
CompressedSize64 uint64
|
||||
UncompressedSize64 uint64
|
||||
Extra []byte
|
||||
ExternalAttrs uint32 // Meaning depends on CreatorVersion
|
||||
}
|
||||
|
||||
// FileInfo returns an fs.FileInfo for the FileHeader.
|
||||
func (h *FileHeader) FileInfo() fs.FileInfo {
|
||||
return headerFileInfo{h}
|
||||
}
|
||||
|
||||
// headerFileInfo implements fs.FileInfo.
|
||||
type headerFileInfo struct {
|
||||
fh *FileHeader
|
||||
}
|
||||
|
||||
func (fi headerFileInfo) Name() string { return path.Base(fi.fh.Name) }
|
||||
func (fi headerFileInfo) Size() int64 {
|
||||
if fi.fh.UncompressedSize64 > 0 {
|
||||
return int64(fi.fh.UncompressedSize64)
|
||||
}
|
||||
return int64(fi.fh.UncompressedSize)
|
||||
}
|
||||
func (fi headerFileInfo) IsDir() bool { return fi.Mode().IsDir() }
|
||||
func (fi headerFileInfo) ModTime() time.Time {
|
||||
if fi.fh.Modified.IsZero() {
|
||||
return fi.fh.ModTime()
|
||||
}
|
||||
return fi.fh.Modified.UTC()
|
||||
}
|
||||
func (fi headerFileInfo) Mode() fs.FileMode { return fi.fh.Mode() }
|
||||
func (fi headerFileInfo) Type() fs.FileMode { return fi.fh.Mode().Type() }
|
||||
func (fi headerFileInfo) Sys() interface{} { return fi.fh }
|
||||
|
||||
func (fi headerFileInfo) Info() (fs.FileInfo, error) { return fi, nil }
|
||||
|
||||
// FileInfoHeader creates a partially-populated FileHeader from an
|
||||
// fs.FileInfo.
|
||||
// Because fs.FileInfo's Name method returns only the base name of
|
||||
// the file it describes, it may be necessary to modify the Name field
|
||||
// of the returned header to provide the full path name of the file.
|
||||
// If compression is desired, callers should set the FileHeader.Method
|
||||
// field; it is unset by default.
|
||||
func FileInfoHeader(fi fs.FileInfo) (*FileHeader, error) {
|
||||
size := fi.Size()
|
||||
fh := &FileHeader{
|
||||
Name: fi.Name(),
|
||||
UncompressedSize64: uint64(size),
|
||||
}
|
||||
fh.SetModTime(fi.ModTime())
|
||||
fh.SetMode(fi.Mode())
|
||||
if fh.UncompressedSize64 > uint32max {
|
||||
fh.UncompressedSize = uint32max
|
||||
} else {
|
||||
fh.UncompressedSize = uint32(fh.UncompressedSize64)
|
||||
}
|
||||
return fh, nil
|
||||
}
|
||||
|
||||
type directoryEnd struct {
|
||||
diskNbr uint32 // unused
|
||||
dirDiskNbr uint32 // unused
|
||||
dirRecordsThisDisk uint64 // unused
|
||||
directoryRecords uint64
|
||||
directorySize uint64
|
||||
directoryOffset uint64 // relative to file
|
||||
commentLen uint16
|
||||
comment string
|
||||
}
|
||||
|
||||
// timeZone returns a *time.Location based on the provided offset.
|
||||
// If the offset is non-sensible, then this uses an offset of zero.
|
||||
func timeZone(offset time.Duration) *time.Location {
|
||||
const (
|
||||
minOffset = -12 * time.Hour // E.g., Baker island at -12:00
|
||||
maxOffset = +14 * time.Hour // E.g., Line island at +14:00
|
||||
offsetAlias = 15 * time.Minute // E.g., Nepal at +5:45
|
||||
)
|
||||
offset = offset.Round(offsetAlias)
|
||||
if offset < minOffset || maxOffset < offset {
|
||||
offset = 0
|
||||
}
|
||||
return time.FixedZone("", int(offset/time.Second))
|
||||
}
|
||||
|
||||
// msDosTimeToTime converts an MS-DOS date and time into a time.Time.
|
||||
// The resolution is 2s.
|
||||
// See: https://msdn.microsoft.com/en-us/library/ms724247(v=VS.85).aspx
|
||||
func msDosTimeToTime(dosDate, dosTime uint16) time.Time {
|
||||
return time.Date(
|
||||
// date bits 0-4: day of month; 5-8: month; 9-15: years since 1980
|
||||
int(dosDate>>9+1980),
|
||||
time.Month(dosDate>>5&0xf),
|
||||
int(dosDate&0x1f),
|
||||
|
||||
// time bits 0-4: second/2; 5-10: minute; 11-15: hour
|
||||
int(dosTime>>11),
|
||||
int(dosTime>>5&0x3f),
|
||||
int(dosTime&0x1f*2),
|
||||
0, // nanoseconds
|
||||
|
||||
time.UTC,
|
||||
)
|
||||
}
|
||||
|
||||
// timeToMsDosTime converts a time.Time to an MS-DOS date and time.
|
||||
// The resolution is 2s.
|
||||
// See: https://msdn.microsoft.com/en-us/library/ms724274(v=VS.85).aspx
|
||||
func timeToMsDosTime(t time.Time) (fDate uint16, fTime uint16) {
|
||||
fDate = uint16(t.Day() + int(t.Month())<<5 + (t.Year()-1980)<<9)
|
||||
fTime = uint16(t.Second()/2 + t.Minute()<<5 + t.Hour()<<11)
|
||||
return
|
||||
}
|
||||
|
||||
// ModTime returns the modification time in UTC using the legacy
|
||||
// ModifiedDate and ModifiedTime fields.
|
||||
//
|
||||
// Deprecated: Use Modified instead.
|
||||
func (h *FileHeader) ModTime() time.Time {
|
||||
return msDosTimeToTime(h.ModifiedDate, h.ModifiedTime)
|
||||
}
|
||||
|
||||
// SetModTime sets the Modified, ModifiedTime, and ModifiedDate fields
|
||||
// to the given time in UTC.
|
||||
//
|
||||
// Deprecated: Use Modified instead.
|
||||
func (h *FileHeader) SetModTime(t time.Time) {
|
||||
t = t.UTC() // Convert to UTC for compatibility
|
||||
h.Modified = t
|
||||
h.ModifiedDate, h.ModifiedTime = timeToMsDosTime(t)
|
||||
}
|
||||
|
||||
const (
|
||||
// Unix constants. The specification doesn't mention them,
|
||||
// but these seem to be the values agreed on by tools.
|
||||
s_IFMT = 0xf000
|
||||
s_IFSOCK = 0xc000
|
||||
s_IFLNK = 0xa000
|
||||
s_IFREG = 0x8000
|
||||
s_IFBLK = 0x6000
|
||||
s_IFDIR = 0x4000
|
||||
s_IFCHR = 0x2000
|
||||
s_IFIFO = 0x1000
|
||||
s_ISUID = 0x800
|
||||
s_ISGID = 0x400
|
||||
s_ISVTX = 0x200
|
||||
|
||||
msdosDir = 0x10
|
||||
msdosReadOnly = 0x01
|
||||
)
|
||||
|
||||
// Mode returns the permission and mode bits for the FileHeader.
|
||||
func (h *FileHeader) Mode() (mode fs.FileMode) {
|
||||
switch h.CreatorVersion >> 8 {
|
||||
case creatorUnix, creatorMacOSX:
|
||||
mode = unixModeToFileMode(h.ExternalAttrs >> 16)
|
||||
case creatorNTFS, creatorVFAT, creatorFAT:
|
||||
mode = msdosModeToFileMode(h.ExternalAttrs)
|
||||
}
|
||||
if len(h.Name) > 0 && h.Name[len(h.Name)-1] == '/' {
|
||||
mode |= fs.ModeDir
|
||||
}
|
||||
return mode
|
||||
}
|
||||
|
||||
// SetMode changes the permission and mode bits for the FileHeader.
|
||||
func (h *FileHeader) SetMode(mode fs.FileMode) {
|
||||
h.CreatorVersion = h.CreatorVersion&0xff | creatorUnix<<8
|
||||
h.ExternalAttrs = fileModeToUnixMode(mode) << 16
|
||||
|
||||
// set MSDOS attributes too, as the original zip does.
|
||||
if mode&fs.ModeDir != 0 {
|
||||
h.ExternalAttrs |= msdosDir
|
||||
}
|
||||
if mode&0200 == 0 {
|
||||
h.ExternalAttrs |= msdosReadOnly
|
||||
}
|
||||
}
|
||||
|
||||
// isZip64 reports whether the file size exceeds the 32 bit limit
|
||||
func (h *FileHeader) isZip64() bool {
|
||||
return h.CompressedSize64 >= uint32max || h.UncompressedSize64 >= uint32max
|
||||
}
|
||||
|
||||
func (f *FileHeader) hasDataDescriptor() bool {
|
||||
return f.Flags&0x8 != 0
|
||||
}
|
||||
|
||||
func msdosModeToFileMode(m uint32) (mode fs.FileMode) {
|
||||
if m&msdosDir != 0 {
|
||||
mode = fs.ModeDir | 0777
|
||||
} else {
|
||||
mode = 0666
|
||||
}
|
||||
if m&msdosReadOnly != 0 {
|
||||
mode &^= 0222
|
||||
}
|
||||
return mode
|
||||
}
|
||||
|
||||
func fileModeToUnixMode(mode fs.FileMode) uint32 {
|
||||
var m uint32
|
||||
switch mode & fs.ModeType {
|
||||
default:
|
||||
m = s_IFREG
|
||||
case fs.ModeDir:
|
||||
m = s_IFDIR
|
||||
case fs.ModeSymlink:
|
||||
m = s_IFLNK
|
||||
case fs.ModeNamedPipe:
|
||||
m = s_IFIFO
|
||||
case fs.ModeSocket:
|
||||
m = s_IFSOCK
|
||||
case fs.ModeDevice:
|
||||
m = s_IFBLK
|
||||
case fs.ModeDevice | fs.ModeCharDevice:
|
||||
m = s_IFCHR
|
||||
}
|
||||
if mode&fs.ModeSetuid != 0 {
|
||||
m |= s_ISUID
|
||||
}
|
||||
if mode&fs.ModeSetgid != 0 {
|
||||
m |= s_ISGID
|
||||
}
|
||||
if mode&fs.ModeSticky != 0 {
|
||||
m |= s_ISVTX
|
||||
}
|
||||
return m | uint32(mode&0777)
|
||||
}
|
||||
|
||||
func unixModeToFileMode(m uint32) fs.FileMode {
|
||||
mode := fs.FileMode(m & 0777)
|
||||
switch m & s_IFMT {
|
||||
case s_IFBLK:
|
||||
mode |= fs.ModeDevice
|
||||
case s_IFCHR:
|
||||
mode |= fs.ModeDevice | fs.ModeCharDevice
|
||||
case s_IFDIR:
|
||||
mode |= fs.ModeDir
|
||||
case s_IFIFO:
|
||||
mode |= fs.ModeNamedPipe
|
||||
case s_IFLNK:
|
||||
mode |= fs.ModeSymlink
|
||||
case s_IFREG:
|
||||
// nothing to do
|
||||
case s_IFSOCK:
|
||||
mode |= fs.ModeSocket
|
||||
}
|
||||
if m&s_ISGID != 0 {
|
||||
mode |= fs.ModeSetgid
|
||||
}
|
||||
if m&s_ISUID != 0 {
|
||||
mode |= fs.ModeSetuid
|
||||
}
|
||||
if m&s_ISVTX != 0 {
|
||||
mode |= fs.ModeSticky
|
||||
}
|
||||
return mode
|
||||
}
|
||||
|
||||
// dataDescriptor holds the data descriptor that optionally follows the file
|
||||
// contents in the zip file.
|
||||
type dataDescriptor struct {
|
||||
crc32 uint32
|
||||
compressedSize uint64
|
||||
uncompressedSize uint64
|
||||
}
|
||||
390
vendor/github.com/klauspost/compress/zip/struct_leg.go
сгенерированный
поставляемый
Обычный файл
390
vendor/github.com/klauspost/compress/zip/struct_leg.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,390 @@
|
||||
// Copyright 2010 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
//go:build !go1.16
|
||||
// +build !go1.16
|
||||
|
||||
/*
|
||||
Package zip provides support for reading and writing ZIP archives.
|
||||
|
||||
See: https://www.pkware.com/appnote
|
||||
|
||||
This package does not support disk spanning.
|
||||
|
||||
A note about ZIP64:
|
||||
|
||||
To be backwards compatible the FileHeader has both 32 and 64 bit Size
|
||||
fields. The 64 bit fields will always contain the correct value and
|
||||
for normal archives both fields will be the same. For files requiring
|
||||
the ZIP64 format the 32 bit fields will be 0xffffffff and the 64 bit
|
||||
fields must be used instead.
|
||||
*/
|
||||
package zip
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Compression methods.
|
||||
const (
|
||||
Store uint16 = 0 // no compression
|
||||
Deflate uint16 = 8 // DEFLATE compressed
|
||||
)
|
||||
|
||||
const (
|
||||
fileHeaderSignature = 0x04034b50
|
||||
directoryHeaderSignature = 0x02014b50
|
||||
directoryEndSignature = 0x06054b50
|
||||
directory64LocSignature = 0x07064b50
|
||||
directory64EndSignature = 0x06064b50
|
||||
dataDescriptorSignature = 0x08074b50 // de-facto standard; required by OS X Finder
|
||||
fileHeaderLen = 30 // + filename + extra
|
||||
directoryHeaderLen = 46 // + filename + extra + comment
|
||||
directoryEndLen = 22 // + comment
|
||||
dataDescriptorLen = 16 // four uint32: descriptor signature, crc32, compressed size, size
|
||||
dataDescriptor64Len = 24 // descriptor with 8 byte sizes
|
||||
directory64LocLen = 20 //
|
||||
directory64EndLen = 56 // + extra
|
||||
|
||||
// Constants for the first byte in CreatorVersion.
|
||||
creatorFAT = 0
|
||||
creatorUnix = 3
|
||||
creatorNTFS = 11
|
||||
creatorVFAT = 14
|
||||
creatorMacOSX = 19
|
||||
|
||||
// Version numbers.
|
||||
zipVersion20 = 20 // 2.0
|
||||
zipVersion45 = 45 // 4.5 (reads and writes zip64 archives)
|
||||
|
||||
// Limits for non zip64 files.
|
||||
uint16max = (1 << 16) - 1
|
||||
uint32max = (1 << 32) - 1
|
||||
|
||||
// Extra header IDs.
|
||||
//
|
||||
// IDs 0..31 are reserved for official use by PKWARE.
|
||||
// IDs above that range are defined by third-party vendors.
|
||||
// Since ZIP lacked high precision timestamps (nor a official specification
|
||||
// of the timezone used for the date fields), many competing extra fields
|
||||
// have been invented. Pervasive use effectively makes them "official".
|
||||
//
|
||||
// See http://mdfs.net/Docs/Comp/Archiving/Zip/ExtraField
|
||||
zip64ExtraID = 0x0001 // Zip64 extended information
|
||||
ntfsExtraID = 0x000a // NTFS
|
||||
unixExtraID = 0x000d // UNIX
|
||||
extTimeExtraID = 0x5455 // Extended timestamp
|
||||
infoZipUnixExtraID = 0x5855 // Info-ZIP Unix extension
|
||||
)
|
||||
|
||||
// FileHeader describes a file within a zip file.
|
||||
// See the zip spec for details.
|
||||
type FileHeader struct {
|
||||
// Name is the name of the file.
|
||||
//
|
||||
// It must be a relative path, not start with a drive letter (such as "C:"),
|
||||
// and must use forward slashes instead of back slashes. A trailing slash
|
||||
// indicates that this file is a directory and should have no data.
|
||||
//
|
||||
// When reading zip files, the Name field is populated from
|
||||
// the zip file directly and is not validated for correctness.
|
||||
// It is the caller's responsibility to sanitize it as
|
||||
// appropriate, including canonicalizing slash directions,
|
||||
// validating that paths are relative, and preventing path
|
||||
// traversal through filenames ("../../../").
|
||||
Name string
|
||||
|
||||
// Comment is any arbitrary user-defined string shorter than 64KiB.
|
||||
Comment string
|
||||
|
||||
// NonUTF8 indicates that Name and Comment are not encoded in UTF-8.
|
||||
//
|
||||
// By specification, the only other encoding permitted should be CP-437,
|
||||
// but historically many ZIP readers interpret Name and Comment as whatever
|
||||
// the system's local character encoding happens to be.
|
||||
//
|
||||
// This flag should only be set if the user intends to encode a non-portable
|
||||
// ZIP file for a specific localized region. Otherwise, the Writer
|
||||
// automatically sets the ZIP format's UTF-8 flag for valid UTF-8 strings.
|
||||
NonUTF8 bool
|
||||
|
||||
CreatorVersion uint16
|
||||
ReaderVersion uint16
|
||||
Flags uint16
|
||||
|
||||
// Method is the compression method. If zero, Store is used.
|
||||
Method uint16
|
||||
|
||||
// Modified is the modified time of the file.
|
||||
//
|
||||
// When reading, an extended timestamp is preferred over the legacy MS-DOS
|
||||
// date field, and the offset between the times is used as the timezone.
|
||||
// If only the MS-DOS date is present, the timezone is assumed to be UTC.
|
||||
//
|
||||
// When writing, an extended timestamp (which is timezone-agnostic) is
|
||||
// always emitted. The legacy MS-DOS date field is encoded according to the
|
||||
// location of the Modified time.
|
||||
Modified time.Time
|
||||
ModifiedTime uint16 // Deprecated: Legacy MS-DOS date; use Modified instead.
|
||||
ModifiedDate uint16 // Deprecated: Legacy MS-DOS time; use Modified instead.
|
||||
|
||||
CRC32 uint32
|
||||
CompressedSize uint32 // Deprecated: Use CompressedSize64 instead.
|
||||
UncompressedSize uint32 // Deprecated: Use UncompressedSize64 instead.
|
||||
CompressedSize64 uint64
|
||||
UncompressedSize64 uint64
|
||||
Extra []byte
|
||||
ExternalAttrs uint32 // Meaning depends on CreatorVersion
|
||||
}
|
||||
|
||||
// FileInfo returns an os.FileInfo for the FileHeader.
|
||||
func (h *FileHeader) FileInfo() os.FileInfo {
|
||||
return headerFileInfo{h}
|
||||
}
|
||||
|
||||
// headerFileInfo implements os.FileInfo.
|
||||
type headerFileInfo struct {
|
||||
fh *FileHeader
|
||||
}
|
||||
|
||||
func (fi headerFileInfo) Name() string { return path.Base(fi.fh.Name) }
|
||||
func (fi headerFileInfo) Size() int64 {
|
||||
if fi.fh.UncompressedSize64 > 0 {
|
||||
return int64(fi.fh.UncompressedSize64)
|
||||
}
|
||||
return int64(fi.fh.UncompressedSize)
|
||||
}
|
||||
func (fi headerFileInfo) IsDir() bool { return fi.Mode().IsDir() }
|
||||
func (fi headerFileInfo) ModTime() time.Time {
|
||||
if fi.fh.Modified.IsZero() {
|
||||
return fi.fh.ModTime()
|
||||
}
|
||||
return fi.fh.Modified.UTC()
|
||||
}
|
||||
func (fi headerFileInfo) Mode() os.FileMode { return fi.fh.Mode() }
|
||||
func (fi headerFileInfo) Sys() interface{} { return fi.fh }
|
||||
|
||||
// FileInfoHeader creates a partially-populated FileHeader from an
|
||||
// os.FileInfo.
|
||||
// Because os.FileInfo's Name method returns only the base name of
|
||||
// the file it describes, it may be necessary to modify the Name field
|
||||
// of the returned header to provide the full path name of the file.
|
||||
// If compression is desired, callers should set the FileHeader.Method
|
||||
// field; it is unset by default.
|
||||
func FileInfoHeader(fi os.FileInfo) (*FileHeader, error) {
|
||||
size := fi.Size()
|
||||
fh := &FileHeader{
|
||||
Name: fi.Name(),
|
||||
UncompressedSize64: uint64(size),
|
||||
}
|
||||
fh.SetModTime(fi.ModTime())
|
||||
fh.SetMode(fi.Mode())
|
||||
if fh.UncompressedSize64 > uint32max {
|
||||
fh.UncompressedSize = uint32max
|
||||
} else {
|
||||
fh.UncompressedSize = uint32(fh.UncompressedSize64)
|
||||
}
|
||||
return fh, nil
|
||||
}
|
||||
|
||||
type directoryEnd struct {
|
||||
diskNbr uint32 // unused
|
||||
dirDiskNbr uint32 // unused
|
||||
dirRecordsThisDisk uint64 // unused
|
||||
directoryRecords uint64
|
||||
directorySize uint64
|
||||
directoryOffset uint64 // relative to file
|
||||
commentLen uint16
|
||||
comment string
|
||||
}
|
||||
|
||||
// timeZone returns a *time.Location based on the provided offset.
|
||||
// If the offset is non-sensible, then this uses an offset of zero.
|
||||
func timeZone(offset time.Duration) *time.Location {
|
||||
const (
|
||||
minOffset = -12 * time.Hour // E.g., Baker island at -12:00
|
||||
maxOffset = +14 * time.Hour // E.g., Line island at +14:00
|
||||
offsetAlias = 15 * time.Minute // E.g., Nepal at +5:45
|
||||
)
|
||||
offset = offset.Round(offsetAlias)
|
||||
if offset < minOffset || maxOffset < offset {
|
||||
offset = 0
|
||||
}
|
||||
return time.FixedZone("", int(offset/time.Second))
|
||||
}
|
||||
|
||||
// msDosTimeToTime converts an MS-DOS date and time into a time.Time.
|
||||
// The resolution is 2s.
|
||||
// See: https://msdn.microsoft.com/en-us/library/ms724247(v=VS.85).aspx
|
||||
func msDosTimeToTime(dosDate, dosTime uint16) time.Time {
|
||||
return time.Date(
|
||||
// date bits 0-4: day of month; 5-8: month; 9-15: years since 1980
|
||||
int(dosDate>>9+1980),
|
||||
time.Month(dosDate>>5&0xf),
|
||||
int(dosDate&0x1f),
|
||||
|
||||
// time bits 0-4: second/2; 5-10: minute; 11-15: hour
|
||||
int(dosTime>>11),
|
||||
int(dosTime>>5&0x3f),
|
||||
int(dosTime&0x1f*2),
|
||||
0, // nanoseconds
|
||||
|
||||
time.UTC,
|
||||
)
|
||||
}
|
||||
|
||||
// timeToMsDosTime converts a time.Time to an MS-DOS date and time.
|
||||
// The resolution is 2s.
|
||||
// See: https://msdn.microsoft.com/en-us/library/ms724274(v=VS.85).aspx
|
||||
func timeToMsDosTime(t time.Time) (fDate uint16, fTime uint16) {
|
||||
fDate = uint16(t.Day() + int(t.Month())<<5 + (t.Year()-1980)<<9)
|
||||
fTime = uint16(t.Second()/2 + t.Minute()<<5 + t.Hour()<<11)
|
||||
return
|
||||
}
|
||||
|
||||
// ModTime returns the modification time in UTC using the legacy
|
||||
// ModifiedDate and ModifiedTime fields.
|
||||
//
|
||||
// Deprecated: Use Modified instead.
|
||||
func (h *FileHeader) ModTime() time.Time {
|
||||
return msDosTimeToTime(h.ModifiedDate, h.ModifiedTime)
|
||||
}
|
||||
|
||||
// SetModTime sets the Modified, ModifiedTime, and ModifiedDate fields
|
||||
// to the given time in UTC.
|
||||
//
|
||||
// Deprecated: Use Modified instead.
|
||||
func (h *FileHeader) SetModTime(t time.Time) {
|
||||
t = t.UTC() // Convert to UTC for compatibility
|
||||
h.Modified = t
|
||||
h.ModifiedDate, h.ModifiedTime = timeToMsDosTime(t)
|
||||
}
|
||||
|
||||
const (
|
||||
// Unix constants. The specification doesn't mention them,
|
||||
// but these seem to be the values agreed on by tools.
|
||||
s_IFMT = 0xf000
|
||||
s_IFSOCK = 0xc000
|
||||
s_IFLNK = 0xa000
|
||||
s_IFREG = 0x8000
|
||||
s_IFBLK = 0x6000
|
||||
s_IFDIR = 0x4000
|
||||
s_IFCHR = 0x2000
|
||||
s_IFIFO = 0x1000
|
||||
s_ISUID = 0x800
|
||||
s_ISGID = 0x400
|
||||
s_ISVTX = 0x200
|
||||
|
||||
msdosDir = 0x10
|
||||
msdosReadOnly = 0x01
|
||||
)
|
||||
|
||||
// Mode returns the permission and mode bits for the FileHeader.
|
||||
func (h *FileHeader) Mode() (mode os.FileMode) {
|
||||
switch h.CreatorVersion >> 8 {
|
||||
case creatorUnix, creatorMacOSX:
|
||||
mode = unixModeToFileMode(h.ExternalAttrs >> 16)
|
||||
case creatorNTFS, creatorVFAT, creatorFAT:
|
||||
mode = msdosModeToFileMode(h.ExternalAttrs)
|
||||
}
|
||||
if len(h.Name) > 0 && h.Name[len(h.Name)-1] == '/' {
|
||||
mode |= os.ModeDir
|
||||
}
|
||||
return mode
|
||||
}
|
||||
|
||||
// SetMode changes the permission and mode bits for the FileHeader.
|
||||
func (h *FileHeader) SetMode(mode os.FileMode) {
|
||||
h.CreatorVersion = h.CreatorVersion&0xff | creatorUnix<<8
|
||||
h.ExternalAttrs = fileModeToUnixMode(mode) << 16
|
||||
|
||||
// set MSDOS attributes too, as the original zip does.
|
||||
if mode&os.ModeDir != 0 {
|
||||
h.ExternalAttrs |= msdosDir
|
||||
}
|
||||
if mode&0200 == 0 {
|
||||
h.ExternalAttrs |= msdosReadOnly
|
||||
}
|
||||
}
|
||||
|
||||
// isZip64 reports whether the file size exceeds the 32 bit limit
|
||||
func (h *FileHeader) isZip64() bool {
|
||||
return h.CompressedSize64 >= uint32max || h.UncompressedSize64 >= uint32max
|
||||
}
|
||||
|
||||
func msdosModeToFileMode(m uint32) (mode os.FileMode) {
|
||||
if m&msdosDir != 0 {
|
||||
mode = os.ModeDir | 0777
|
||||
} else {
|
||||
mode = 0666
|
||||
}
|
||||
if m&msdosReadOnly != 0 {
|
||||
mode &^= 0222
|
||||
}
|
||||
return mode
|
||||
}
|
||||
|
||||
func fileModeToUnixMode(mode os.FileMode) uint32 {
|
||||
var m uint32
|
||||
switch mode & os.ModeType {
|
||||
default:
|
||||
m = s_IFREG
|
||||
case os.ModeDir:
|
||||
m = s_IFDIR
|
||||
case os.ModeSymlink:
|
||||
m = s_IFLNK
|
||||
case os.ModeNamedPipe:
|
||||
m = s_IFIFO
|
||||
case os.ModeSocket:
|
||||
m = s_IFSOCK
|
||||
case os.ModeDevice:
|
||||
if mode&os.ModeCharDevice != 0 {
|
||||
m = s_IFCHR
|
||||
} else {
|
||||
m = s_IFBLK
|
||||
}
|
||||
}
|
||||
if mode&os.ModeSetuid != 0 {
|
||||
m |= s_ISUID
|
||||
}
|
||||
if mode&os.ModeSetgid != 0 {
|
||||
m |= s_ISGID
|
||||
}
|
||||
if mode&os.ModeSticky != 0 {
|
||||
m |= s_ISVTX
|
||||
}
|
||||
return m | uint32(mode&0777)
|
||||
}
|
||||
|
||||
func unixModeToFileMode(m uint32) os.FileMode {
|
||||
mode := os.FileMode(m & 0777)
|
||||
switch m & s_IFMT {
|
||||
case s_IFBLK:
|
||||
mode |= os.ModeDevice
|
||||
case s_IFCHR:
|
||||
mode |= os.ModeDevice | os.ModeCharDevice
|
||||
case s_IFDIR:
|
||||
mode |= os.ModeDir
|
||||
case s_IFIFO:
|
||||
mode |= os.ModeNamedPipe
|
||||
case s_IFLNK:
|
||||
mode |= os.ModeSymlink
|
||||
case s_IFREG:
|
||||
// nothing to do
|
||||
case s_IFSOCK:
|
||||
mode |= os.ModeSocket
|
||||
}
|
||||
if m&s_ISGID != 0 {
|
||||
mode |= os.ModeSetgid
|
||||
}
|
||||
if m&s_ISUID != 0 {
|
||||
mode |= os.ModeSetuid
|
||||
}
|
||||
if m&s_ISVTX != 0 {
|
||||
mode |= os.ModeSticky
|
||||
}
|
||||
return mode
|
||||
}
|
||||
643
vendor/github.com/klauspost/compress/zip/writer.go
сгенерированный
поставляемый
Обычный файл
643
vendor/github.com/klauspost/compress/zip/writer.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,643 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
//go:build go1.16
|
||||
// +build go1.16
|
||||
|
||||
package zip
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"hash"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
var (
|
||||
errLongName = errors.New("zip: FileHeader.Name too long")
|
||||
errLongExtra = errors.New("zip: FileHeader.Extra too long")
|
||||
)
|
||||
|
||||
// Writer implements a zip file writer.
|
||||
type Writer struct {
|
||||
cw *countWriter
|
||||
dir []*header
|
||||
last *fileWriter
|
||||
closed bool
|
||||
compressors map[uint16]Compressor
|
||||
comment string
|
||||
|
||||
// testHookCloseSizeOffset if non-nil is called with the size
|
||||
// of offset of the central directory at Close.
|
||||
testHookCloseSizeOffset func(size, offset uint64)
|
||||
}
|
||||
|
||||
type header struct {
|
||||
*FileHeader
|
||||
offset uint64
|
||||
raw bool
|
||||
}
|
||||
|
||||
// NewWriter returns a new Writer writing a zip file to w.
|
||||
func NewWriter(w io.Writer) *Writer {
|
||||
return &Writer{cw: &countWriter{w: bufio.NewWriter(w)}}
|
||||
}
|
||||
|
||||
// SetOffset sets the offset of the beginning of the zip data within the
|
||||
// underlying writer. It should be used when the zip data is appended to an
|
||||
// existing file, such as a binary executable.
|
||||
// It must be called before any data is written.
|
||||
func (w *Writer) SetOffset(n int64) {
|
||||
if w.cw.count != 0 {
|
||||
panic("zip: SetOffset called after data was written")
|
||||
}
|
||||
w.cw.count = n
|
||||
}
|
||||
|
||||
// Flush flushes any buffered data to the underlying writer.
|
||||
// Calling Flush is not normally necessary; calling Close is sufficient.
|
||||
func (w *Writer) Flush() error {
|
||||
return w.cw.w.(*bufio.Writer).Flush()
|
||||
}
|
||||
|
||||
// SetComment sets the end-of-central-directory comment field.
|
||||
// It can only be called before Close.
|
||||
func (w *Writer) SetComment(comment string) error {
|
||||
if len(comment) > uint16max {
|
||||
return errors.New("zip: Writer.Comment too long")
|
||||
}
|
||||
w.comment = comment
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close finishes writing the zip file by writing the central directory.
|
||||
// It does not close the underlying writer.
|
||||
func (w *Writer) Close() error {
|
||||
if w.last != nil && !w.last.closed {
|
||||
if err := w.last.close(); err != nil {
|
||||
return err
|
||||
}
|
||||
w.last = nil
|
||||
}
|
||||
if w.closed {
|
||||
return errors.New("zip: writer closed twice")
|
||||
}
|
||||
w.closed = true
|
||||
|
||||
// write central directory
|
||||
start := w.cw.count
|
||||
for _, h := range w.dir {
|
||||
var buf [directoryHeaderLen]byte
|
||||
b := writeBuf(buf[:])
|
||||
b.uint32(uint32(directoryHeaderSignature))
|
||||
b.uint16(h.CreatorVersion)
|
||||
b.uint16(h.ReaderVersion)
|
||||
b.uint16(h.Flags)
|
||||
b.uint16(h.Method)
|
||||
b.uint16(h.ModifiedTime)
|
||||
b.uint16(h.ModifiedDate)
|
||||
b.uint32(h.CRC32)
|
||||
if h.isZip64() || h.offset >= uint32max {
|
||||
// the file needs a zip64 header. store maxint in both
|
||||
// 32 bit size fields (and offset later) to signal that the
|
||||
// zip64 extra header should be used.
|
||||
b.uint32(uint32max) // compressed size
|
||||
b.uint32(uint32max) // uncompressed size
|
||||
|
||||
// append a zip64 extra block to Extra
|
||||
var buf [28]byte // 2x uint16 + 3x uint64
|
||||
eb := writeBuf(buf[:])
|
||||
eb.uint16(zip64ExtraID)
|
||||
eb.uint16(24) // size = 3x uint64
|
||||
eb.uint64(h.UncompressedSize64)
|
||||
eb.uint64(h.CompressedSize64)
|
||||
eb.uint64(h.offset)
|
||||
h.Extra = append(h.Extra, buf[:]...)
|
||||
} else {
|
||||
b.uint32(h.CompressedSize)
|
||||
b.uint32(h.UncompressedSize)
|
||||
}
|
||||
|
||||
b.uint16(uint16(len(h.Name)))
|
||||
b.uint16(uint16(len(h.Extra)))
|
||||
b.uint16(uint16(len(h.Comment)))
|
||||
b = b[4:] // skip disk number start and internal file attr (2x uint16)
|
||||
b.uint32(h.ExternalAttrs)
|
||||
if h.offset > uint32max {
|
||||
b.uint32(uint32max)
|
||||
} else {
|
||||
b.uint32(uint32(h.offset))
|
||||
}
|
||||
if _, err := w.cw.Write(buf[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.WriteString(w.cw, h.Name); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := w.cw.Write(h.Extra); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.WriteString(w.cw, h.Comment); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
end := w.cw.count
|
||||
|
||||
records := uint64(len(w.dir))
|
||||
size := uint64(end - start)
|
||||
offset := uint64(start)
|
||||
|
||||
if f := w.testHookCloseSizeOffset; f != nil {
|
||||
f(size, offset)
|
||||
}
|
||||
|
||||
if records >= uint16max || size >= uint32max || offset >= uint32max {
|
||||
var buf [directory64EndLen + directory64LocLen]byte
|
||||
b := writeBuf(buf[:])
|
||||
|
||||
// zip64 end of central directory record
|
||||
b.uint32(directory64EndSignature)
|
||||
b.uint64(directory64EndLen - 12) // length minus signature (uint32) and length fields (uint64)
|
||||
b.uint16(zipVersion45) // version made by
|
||||
b.uint16(zipVersion45) // version needed to extract
|
||||
b.uint32(0) // number of this disk
|
||||
b.uint32(0) // number of the disk with the start of the central directory
|
||||
b.uint64(records) // total number of entries in the central directory on this disk
|
||||
b.uint64(records) // total number of entries in the central directory
|
||||
b.uint64(size) // size of the central directory
|
||||
b.uint64(offset) // offset of start of central directory with respect to the starting disk number
|
||||
|
||||
// zip64 end of central directory locator
|
||||
b.uint32(directory64LocSignature)
|
||||
b.uint32(0) // number of the disk with the start of the zip64 end of central directory
|
||||
b.uint64(uint64(end)) // relative offset of the zip64 end of central directory record
|
||||
b.uint32(1) // total number of disks
|
||||
|
||||
if _, err := w.cw.Write(buf[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// store max values in the regular end record to signal
|
||||
// that the zip64 values should be used instead
|
||||
records = uint16max
|
||||
size = uint32max
|
||||
offset = uint32max
|
||||
}
|
||||
|
||||
// write end record
|
||||
var buf [directoryEndLen]byte
|
||||
b := writeBuf(buf[:])
|
||||
b.uint32(uint32(directoryEndSignature))
|
||||
b = b[4:] // skip over disk number and first disk number (2x uint16)
|
||||
b.uint16(uint16(records)) // number of entries this disk
|
||||
b.uint16(uint16(records)) // number of entries total
|
||||
b.uint32(uint32(size)) // size of directory
|
||||
b.uint32(uint32(offset)) // start of directory
|
||||
b.uint16(uint16(len(w.comment))) // byte size of EOCD comment
|
||||
if _, err := w.cw.Write(buf[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.WriteString(w.cw, w.comment); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return w.cw.w.(*bufio.Writer).Flush()
|
||||
}
|
||||
|
||||
// Create adds a file to the zip file using the provided name.
|
||||
// It returns a Writer to which the file contents should be written.
|
||||
// The file contents will be compressed using the Deflate method.
|
||||
// The name must be a relative path: it must not start with a drive
|
||||
// letter (e.g. C:) or leading slash, and only forward slashes are
|
||||
// allowed. To create a directory instead of a file, add a trailing
|
||||
// slash to the name.
|
||||
// The file's contents must be written to the io.Writer before the next
|
||||
// call to Create, CreateHeader, or Close.
|
||||
func (w *Writer) Create(name string) (io.Writer, error) {
|
||||
header := &FileHeader{
|
||||
Name: name,
|
||||
Method: Deflate,
|
||||
}
|
||||
return w.CreateHeader(header)
|
||||
}
|
||||
|
||||
// detectUTF8 reports whether s is a valid UTF-8 string, and whether the string
|
||||
// must be considered UTF-8 encoding (i.e., not compatible with CP-437, ASCII,
|
||||
// or any other common encoding).
|
||||
func detectUTF8(s string) (valid, require bool) {
|
||||
for i := 0; i < len(s); {
|
||||
r, size := utf8.DecodeRuneInString(s[i:])
|
||||
i += size
|
||||
// Officially, ZIP uses CP-437, but many readers use the system's
|
||||
// local character encoding. Most encoding are compatible with a large
|
||||
// subset of CP-437, which itself is ASCII-like.
|
||||
//
|
||||
// Forbid 0x7e and 0x5c since EUC-KR and Shift-JIS replace those
|
||||
// characters with localized currency and overline characters.
|
||||
if r < 0x20 || r > 0x7d || r == 0x5c {
|
||||
if !utf8.ValidRune(r) || (r == utf8.RuneError && size == 1) {
|
||||
return false, false
|
||||
}
|
||||
require = true
|
||||
}
|
||||
}
|
||||
return true, require
|
||||
}
|
||||
|
||||
// prepare performs the bookkeeping operations required at the start of
|
||||
// CreateHeader and CreateRaw.
|
||||
func (w *Writer) prepare(fh *FileHeader) error {
|
||||
if w.last != nil && !w.last.closed {
|
||||
if err := w.last.close(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(w.dir) > 0 && w.dir[len(w.dir)-1].FileHeader == fh {
|
||||
// See https://golang.org/issue/11144 confusion.
|
||||
return errors.New("archive/zip: invalid duplicate FileHeader")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreateHeader adds a file to the zip archive using the provided FileHeader
|
||||
// for the file metadata. Writer takes ownership of fh and may mutate
|
||||
// its fields. The caller must not modify fh after calling CreateHeader.
|
||||
//
|
||||
// This returns a Writer to which the file contents should be written.
|
||||
// The file's contents must be written to the io.Writer before the next
|
||||
// call to Create, CreateHeader, CreateRaw, or Close.
|
||||
func (w *Writer) CreateHeader(fh *FileHeader) (io.Writer, error) {
|
||||
if err := w.prepare(fh); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// The ZIP format has a sad state of affairs regarding character encoding.
|
||||
// Officially, the name and comment fields are supposed to be encoded
|
||||
// in CP-437 (which is mostly compatible with ASCII), unless the UTF-8
|
||||
// flag bit is set. However, there are several problems:
|
||||
//
|
||||
// * Many ZIP readers still do not support UTF-8.
|
||||
// * If the UTF-8 flag is cleared, several readers simply interpret the
|
||||
// name and comment fields as whatever the local system encoding is.
|
||||
//
|
||||
// In order to avoid breaking readers without UTF-8 support,
|
||||
// we avoid setting the UTF-8 flag if the strings are CP-437 compatible.
|
||||
// However, if the strings require multibyte UTF-8 encoding and is a
|
||||
// valid UTF-8 string, then we set the UTF-8 bit.
|
||||
//
|
||||
// For the case, where the user explicitly wants to specify the encoding
|
||||
// as UTF-8, they will need to set the flag bit themselves.
|
||||
utf8Valid1, utf8Require1 := detectUTF8(fh.Name)
|
||||
utf8Valid2, utf8Require2 := detectUTF8(fh.Comment)
|
||||
switch {
|
||||
case fh.NonUTF8:
|
||||
fh.Flags &^= 0x800
|
||||
case (utf8Require1 || utf8Require2) && (utf8Valid1 && utf8Valid2):
|
||||
fh.Flags |= 0x800
|
||||
}
|
||||
|
||||
fh.CreatorVersion = fh.CreatorVersion&0xff00 | zipVersion20 // preserve compatibility byte
|
||||
fh.ReaderVersion = zipVersion20
|
||||
|
||||
// If Modified is set, this takes precedence over MS-DOS timestamp fields.
|
||||
if !fh.Modified.IsZero() {
|
||||
// Contrary to the FileHeader.SetModTime method, we intentionally
|
||||
// do not convert to UTC, because we assume the user intends to encode
|
||||
// the date using the specified timezone. A user may want this control
|
||||
// because many legacy ZIP readers interpret the timestamp according
|
||||
// to the local timezone.
|
||||
//
|
||||
// The timezone is only non-UTC if a user directly sets the Modified
|
||||
// field directly themselves. All other approaches sets UTC.
|
||||
fh.ModifiedDate, fh.ModifiedTime = timeToMsDosTime(fh.Modified)
|
||||
|
||||
// Use "extended timestamp" format since this is what Info-ZIP uses.
|
||||
// Nearly every major ZIP implementation uses a different format,
|
||||
// but at least most seem to be able to understand the other formats.
|
||||
//
|
||||
// This format happens to be identical for both local and central header
|
||||
// if modification time is the only timestamp being encoded.
|
||||
var mbuf [9]byte // 2*SizeOf(uint16) + SizeOf(uint8) + SizeOf(uint32)
|
||||
mt := uint32(fh.Modified.Unix())
|
||||
eb := writeBuf(mbuf[:])
|
||||
eb.uint16(extTimeExtraID)
|
||||
eb.uint16(5) // Size: SizeOf(uint8) + SizeOf(uint32)
|
||||
eb.uint8(1) // Flags: ModTime
|
||||
eb.uint32(mt) // ModTime
|
||||
fh.Extra = append(fh.Extra, mbuf[:]...)
|
||||
}
|
||||
|
||||
var (
|
||||
ow io.Writer
|
||||
fw *fileWriter
|
||||
)
|
||||
h := &header{
|
||||
FileHeader: fh,
|
||||
offset: uint64(w.cw.count),
|
||||
}
|
||||
|
||||
if strings.HasSuffix(fh.Name, "/") {
|
||||
// Set the compression method to Store to ensure data length is truly zero,
|
||||
// which the writeHeader method always encodes for the size fields.
|
||||
// This is necessary as most compression formats have non-zero lengths
|
||||
// even when compressing an empty string.
|
||||
fh.Method = Store
|
||||
fh.Flags &^= 0x8 // we will not write a data descriptor
|
||||
|
||||
// Explicitly clear sizes as they have no meaning for directories.
|
||||
fh.CompressedSize = 0
|
||||
fh.CompressedSize64 = 0
|
||||
fh.UncompressedSize = 0
|
||||
fh.UncompressedSize64 = 0
|
||||
|
||||
ow = dirWriter{}
|
||||
} else {
|
||||
fh.Flags |= 0x8 // we will write a data descriptor
|
||||
|
||||
fw = &fileWriter{
|
||||
zipw: w.cw,
|
||||
compCount: &countWriter{w: w.cw},
|
||||
crc32: crc32.NewIEEE(),
|
||||
}
|
||||
comp := w.compressor(fh.Method)
|
||||
if comp == nil {
|
||||
return nil, ErrAlgorithm
|
||||
}
|
||||
var err error
|
||||
fw.comp, err = comp(fw.compCount)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fw.rawCount = &countWriter{w: fw.comp}
|
||||
fw.header = h
|
||||
ow = fw
|
||||
}
|
||||
w.dir = append(w.dir, h)
|
||||
if err := writeHeader(w.cw, h); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// If we're creating a directory, fw is nil.
|
||||
w.last = fw
|
||||
return ow, nil
|
||||
}
|
||||
|
||||
func writeHeader(w io.Writer, h *header) error {
|
||||
const maxUint16 = 1<<16 - 1
|
||||
if len(h.Name) > maxUint16 {
|
||||
return errLongName
|
||||
}
|
||||
if len(h.Extra) > maxUint16 {
|
||||
return errLongExtra
|
||||
}
|
||||
|
||||
var buf [fileHeaderLen]byte
|
||||
b := writeBuf(buf[:])
|
||||
b.uint32(uint32(fileHeaderSignature))
|
||||
b.uint16(h.ReaderVersion)
|
||||
b.uint16(h.Flags)
|
||||
b.uint16(h.Method)
|
||||
b.uint16(h.ModifiedTime)
|
||||
b.uint16(h.ModifiedDate)
|
||||
// In raw mode (caller does the compression), the values are either
|
||||
// written here or in the trailing data descriptor based on the header
|
||||
// flags.
|
||||
if h.raw && !h.hasDataDescriptor() {
|
||||
b.uint32(h.CRC32)
|
||||
b.uint32(uint32(min64(h.CompressedSize64, uint32max)))
|
||||
b.uint32(uint32(min64(h.UncompressedSize64, uint32max)))
|
||||
} else {
|
||||
// When this package handle the compression, these values are
|
||||
// always written to the trailing data descriptor.
|
||||
b.uint32(0) // crc32
|
||||
b.uint32(0) // compressed size
|
||||
b.uint32(0) // uncompressed size
|
||||
}
|
||||
b.uint16(uint16(len(h.Name)))
|
||||
b.uint16(uint16(len(h.Extra)))
|
||||
if _, err := w.Write(buf[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.WriteString(w, h.Name); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := w.Write(h.Extra)
|
||||
return err
|
||||
}
|
||||
|
||||
func min64(x, y uint64) uint64 {
|
||||
if x < y {
|
||||
return x
|
||||
}
|
||||
return y
|
||||
}
|
||||
|
||||
// Deprecated: CreateHeaderRaw is replaced by CreateRaw (stdlib name).
|
||||
func (w *Writer) CreateHeaderRaw(fh *FileHeader) (io.Writer, error) {
|
||||
return w.CreateHeader(fh)
|
||||
}
|
||||
|
||||
// CreateRaw adds a file to the zip archive using the provided FileHeader and
|
||||
// returns a Writer to which the file contents should be written. The file's
|
||||
// contents must be written to the io.Writer before the next call to Create,
|
||||
// CreateHeader, CreateRaw, or Close.
|
||||
//
|
||||
// In contrast to CreateHeader, the bytes passed to Writer are not compressed.
|
||||
func (w *Writer) CreateRaw(fh *FileHeader) (io.Writer, error) {
|
||||
if err := w.prepare(fh); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fh.CompressedSize = uint32(min64(fh.CompressedSize64, uint32max))
|
||||
fh.UncompressedSize = uint32(min64(fh.UncompressedSize64, uint32max))
|
||||
|
||||
h := &header{
|
||||
FileHeader: fh,
|
||||
offset: uint64(w.cw.count),
|
||||
raw: true,
|
||||
}
|
||||
w.dir = append(w.dir, h)
|
||||
if err := writeHeader(w.cw, h); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if strings.HasSuffix(fh.Name, "/") {
|
||||
w.last = nil
|
||||
return dirWriter{}, nil
|
||||
}
|
||||
|
||||
fw := &fileWriter{
|
||||
header: h,
|
||||
zipw: w.cw,
|
||||
}
|
||||
w.last = fw
|
||||
return fw, nil
|
||||
}
|
||||
|
||||
// Copy copies the file f (obtained from a Reader) into w. It copies the raw
|
||||
// form directly bypassing decompression, compression, and validation.
|
||||
// CHANGE: Optional file name cannot be specified any more due to stdlib api.
|
||||
func (w *Writer) Copy(f *File) error {
|
||||
r, err := f.OpenRaw()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fw, err := w.CreateRaw(&f.FileHeader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = io.Copy(fw, r)
|
||||
return err
|
||||
}
|
||||
|
||||
// RegisterCompressor registers or overrides a custom compressor for a specific
|
||||
// method ID. If a compressor for a given method is not found, Writer will
|
||||
// default to looking up the compressor at the package level.
|
||||
func (w *Writer) RegisterCompressor(method uint16, comp Compressor) {
|
||||
if w.compressors == nil {
|
||||
w.compressors = make(map[uint16]Compressor)
|
||||
}
|
||||
w.compressors[method] = comp
|
||||
}
|
||||
|
||||
func (w *Writer) compressor(method uint16) Compressor {
|
||||
comp := w.compressors[method]
|
||||
if comp == nil {
|
||||
comp = compressor(method)
|
||||
}
|
||||
return comp
|
||||
}
|
||||
|
||||
type dirWriter struct{}
|
||||
|
||||
func (dirWriter) Write(b []byte) (int, error) {
|
||||
if len(b) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
return 0, errors.New("zip: write to directory")
|
||||
}
|
||||
|
||||
type fileWriter struct {
|
||||
*header
|
||||
zipw io.Writer
|
||||
rawCount *countWriter
|
||||
comp io.WriteCloser
|
||||
compCount *countWriter
|
||||
crc32 hash.Hash32
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (w *fileWriter) Write(p []byte) (int, error) {
|
||||
if w.closed {
|
||||
return 0, errors.New("zip: write to closed file")
|
||||
}
|
||||
if w.raw {
|
||||
return w.zipw.Write(p)
|
||||
}
|
||||
w.crc32.Write(p)
|
||||
return w.rawCount.Write(p)
|
||||
}
|
||||
|
||||
func (w *fileWriter) close() error {
|
||||
if w.closed {
|
||||
return errors.New("zip: file closed twice")
|
||||
}
|
||||
w.closed = true
|
||||
if w.raw {
|
||||
return w.writeDataDescriptor()
|
||||
}
|
||||
if err := w.comp.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// update FileHeader
|
||||
fh := w.header.FileHeader
|
||||
fh.CRC32 = w.crc32.Sum32()
|
||||
fh.CompressedSize64 = uint64(w.compCount.count)
|
||||
fh.UncompressedSize64 = uint64(w.rawCount.count)
|
||||
|
||||
if fh.isZip64() {
|
||||
fh.CompressedSize = uint32max
|
||||
fh.UncompressedSize = uint32max
|
||||
fh.ReaderVersion = zipVersion45 // requires 4.5 - File uses ZIP64 format extensions
|
||||
} else {
|
||||
fh.CompressedSize = uint32(fh.CompressedSize64)
|
||||
fh.UncompressedSize = uint32(fh.UncompressedSize64)
|
||||
}
|
||||
|
||||
return w.writeDataDescriptor()
|
||||
}
|
||||
|
||||
func (w *fileWriter) writeDataDescriptor() error {
|
||||
if !w.hasDataDescriptor() {
|
||||
return nil
|
||||
}
|
||||
// Write data descriptor. This is more complicated than one would
|
||||
// think, see e.g. comments in zipfile.c:putextended() and
|
||||
// http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=7073588.
|
||||
// The approach here is to write 8 byte sizes if needed without
|
||||
// adding a zip64 extra in the local header (too late anyway).
|
||||
var buf []byte
|
||||
if w.isZip64() {
|
||||
buf = make([]byte, dataDescriptor64Len)
|
||||
} else {
|
||||
buf = make([]byte, dataDescriptorLen)
|
||||
}
|
||||
b := writeBuf(buf)
|
||||
b.uint32(dataDescriptorSignature) // de-facto standard, required by OS X
|
||||
b.uint32(w.CRC32)
|
||||
if w.isZip64() {
|
||||
b.uint64(w.CompressedSize64)
|
||||
b.uint64(w.UncompressedSize64)
|
||||
} else {
|
||||
b.uint32(w.CompressedSize)
|
||||
b.uint32(w.UncompressedSize)
|
||||
}
|
||||
_, err := w.zipw.Write(buf)
|
||||
return err
|
||||
}
|
||||
|
||||
type countWriter struct {
|
||||
w io.Writer
|
||||
count int64
|
||||
}
|
||||
|
||||
func (w *countWriter) Write(p []byte) (int, error) {
|
||||
n, err := w.w.Write(p)
|
||||
w.count += int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
type nopCloser struct {
|
||||
io.Writer
|
||||
}
|
||||
|
||||
func (w nopCloser) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type writeBuf []byte
|
||||
|
||||
func (b *writeBuf) uint8(v uint8) {
|
||||
(*b)[0] = v
|
||||
*b = (*b)[1:]
|
||||
}
|
||||
|
||||
func (b *writeBuf) uint16(v uint16) {
|
||||
binary.LittleEndian.PutUint16(*b, v)
|
||||
*b = (*b)[2:]
|
||||
}
|
||||
|
||||
func (b *writeBuf) uint32(v uint32) {
|
||||
binary.LittleEndian.PutUint32(*b, v)
|
||||
*b = (*b)[4:]
|
||||
}
|
||||
|
||||
func (b *writeBuf) uint64(v uint64) {
|
||||
binary.LittleEndian.PutUint64(*b, v)
|
||||
*b = (*b)[8:]
|
||||
}
|
||||
752
vendor/github.com/klauspost/compress/zip/writer_leg.go
сгенерированный
поставляемый
Обычный файл
752
vendor/github.com/klauspost/compress/zip/writer_leg.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,752 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
//go:build !go1.16
|
||||
// +build !go1.16
|
||||
|
||||
package zip
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"hash"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
var (
|
||||
errLongName = errors.New("zip: FileHeader.Name too long")
|
||||
errLongExtra = errors.New("zip: FileHeader.Extra too long")
|
||||
)
|
||||
|
||||
type lastWriter interface {
|
||||
Close() error
|
||||
Closed() bool
|
||||
}
|
||||
|
||||
// Writer implements a zip file writer.
|
||||
type Writer struct {
|
||||
cw *countWriter
|
||||
dir []*header
|
||||
last lastWriter
|
||||
closed bool
|
||||
compressors map[uint16]Compressor
|
||||
comment string
|
||||
|
||||
// testHookCloseSizeOffset if non-nil is called with the size
|
||||
// of offset of the central directory at Close.
|
||||
testHookCloseSizeOffset func(size, offset uint64)
|
||||
}
|
||||
|
||||
type header struct {
|
||||
*FileHeader
|
||||
offset uint64
|
||||
}
|
||||
|
||||
// NewWriter returns a new Writer writing a zip file to w.
|
||||
func NewWriter(w io.Writer) *Writer {
|
||||
return &Writer{cw: &countWriter{w: bufio.NewWriter(w)}}
|
||||
}
|
||||
|
||||
// SetOffset sets the offset of the beginning of the zip data within the
|
||||
// underlying writer. It should be used when the zip data is appended to an
|
||||
// existing file, such as a binary executable.
|
||||
// It must be called before any data is written.
|
||||
func (w *Writer) SetOffset(n int64) {
|
||||
if w.cw.count != 0 {
|
||||
panic("zip: SetOffset called after data was written")
|
||||
}
|
||||
w.cw.count = n
|
||||
}
|
||||
|
||||
// Flush flushes any buffered data to the underlying writer.
|
||||
// Calling Flush is not normally necessary; calling Close is sufficient.
|
||||
func (w *Writer) Flush() error {
|
||||
return w.cw.w.(*bufio.Writer).Flush()
|
||||
}
|
||||
|
||||
// SetComment sets the end-of-central-directory comment field.
|
||||
// It can only be called before Close.
|
||||
func (w *Writer) SetComment(comment string) error {
|
||||
if len(comment) > uint16max {
|
||||
return errors.New("zip: Writer.Comment too long")
|
||||
}
|
||||
w.comment = comment
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close finishes writing the zip file by writing the central directory.
|
||||
// It does not Close the underlying writer.
|
||||
func (w *Writer) Close() error {
|
||||
if w.last != nil && !w.last.Closed() {
|
||||
if err := w.last.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
w.last = nil
|
||||
}
|
||||
if w.closed {
|
||||
return errors.New("zip: writer closed twice")
|
||||
}
|
||||
w.closed = true
|
||||
|
||||
// write central directory
|
||||
start := w.cw.count
|
||||
for _, h := range w.dir {
|
||||
var buf [directoryHeaderLen]byte
|
||||
b := writeBuf(buf[:])
|
||||
b.uint32(uint32(directoryHeaderSignature))
|
||||
b.uint16(h.CreatorVersion)
|
||||
b.uint16(h.ReaderVersion)
|
||||
b.uint16(h.Flags)
|
||||
b.uint16(h.Method)
|
||||
b.uint16(h.ModifiedTime)
|
||||
b.uint16(h.ModifiedDate)
|
||||
b.uint32(h.CRC32)
|
||||
if h.isZip64() || h.offset >= uint32max {
|
||||
// the file needs a zip64 header. store maxint in both
|
||||
// 32 bit size fields (and offset later) to signal that the
|
||||
// zip64 extra header should be used.
|
||||
b.uint32(uint32max) // compressed size
|
||||
b.uint32(uint32max) // uncompressed size
|
||||
|
||||
// append a zip64 extra block to Extra
|
||||
var buf [28]byte // 2x uint16 + 3x uint64
|
||||
eb := writeBuf(buf[:])
|
||||
eb.uint16(zip64ExtraID)
|
||||
eb.uint16(24) // size = 3x uint64
|
||||
eb.uint64(h.UncompressedSize64)
|
||||
eb.uint64(h.CompressedSize64)
|
||||
eb.uint64(h.offset)
|
||||
h.Extra = append(h.Extra, buf[:]...)
|
||||
} else {
|
||||
b.uint32(h.CompressedSize)
|
||||
b.uint32(h.UncompressedSize)
|
||||
}
|
||||
|
||||
b.uint16(uint16(len(h.Name)))
|
||||
b.uint16(uint16(len(h.Extra)))
|
||||
b.uint16(uint16(len(h.Comment)))
|
||||
b = b[4:] // skip disk number start and internal file attr (2x uint16)
|
||||
b.uint32(h.ExternalAttrs)
|
||||
if h.isZip64() || h.offset > uint32max {
|
||||
b.uint32(uint32max)
|
||||
} else {
|
||||
b.uint32(uint32(h.offset))
|
||||
}
|
||||
if _, err := w.cw.Write(buf[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.WriteString(w.cw, h.Name); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := w.cw.Write(h.Extra); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.WriteString(w.cw, h.Comment); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
end := w.cw.count
|
||||
|
||||
records := uint64(len(w.dir))
|
||||
size := uint64(end - start)
|
||||
offset := uint64(start)
|
||||
|
||||
if f := w.testHookCloseSizeOffset; f != nil {
|
||||
f(size, offset)
|
||||
}
|
||||
|
||||
if records >= uint16max || size >= uint32max || offset >= uint32max {
|
||||
var buf [directory64EndLen + directory64LocLen]byte
|
||||
b := writeBuf(buf[:])
|
||||
|
||||
// zip64 end of central directory record
|
||||
b.uint32(directory64EndSignature)
|
||||
b.uint64(directory64EndLen - 12) // length minus signature (uint32) and length fields (uint64)
|
||||
b.uint16(zipVersion45) // version made by
|
||||
b.uint16(zipVersion45) // version needed to extract
|
||||
b.uint32(0) // number of this disk
|
||||
b.uint32(0) // number of the disk with the start of the central directory
|
||||
b.uint64(records) // total number of entries in the central directory on this disk
|
||||
b.uint64(records) // total number of entries in the central directory
|
||||
b.uint64(size) // size of the central directory
|
||||
b.uint64(offset) // offset of start of central directory with respect to the starting disk number
|
||||
|
||||
// zip64 end of central directory locator
|
||||
b.uint32(directory64LocSignature)
|
||||
b.uint32(0) // number of the disk with the start of the zip64 end of central directory
|
||||
b.uint64(uint64(end)) // relative offset of the zip64 end of central directory record
|
||||
b.uint32(1) // total number of disks
|
||||
|
||||
if _, err := w.cw.Write(buf[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// store max values in the regular end record to signal
|
||||
// that the zip64 values should be used instead
|
||||
records = uint16max
|
||||
size = uint32max
|
||||
offset = uint32max
|
||||
}
|
||||
|
||||
// write end record
|
||||
var buf [directoryEndLen]byte
|
||||
b := writeBuf(buf[:])
|
||||
b.uint32(uint32(directoryEndSignature))
|
||||
b = b[4:] // skip over disk number and first disk number (2x uint16)
|
||||
b.uint16(uint16(records)) // number of entries this disk
|
||||
b.uint16(uint16(records)) // number of entries total
|
||||
b.uint32(uint32(size)) // size of directory
|
||||
b.uint32(uint32(offset)) // start of directory
|
||||
b.uint16(uint16(len(w.comment))) // byte size of EOCD comment
|
||||
if _, err := w.cw.Write(buf[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.WriteString(w.cw, w.comment); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return w.cw.w.(*bufio.Writer).Flush()
|
||||
}
|
||||
|
||||
// Create adds a file to the zip file using the provided name.
|
||||
// It returns a Writer to which the file contents should be written.
|
||||
// The file contents will be compressed using the Deflate method.
|
||||
// The name must be a relative path: it must not start with a drive
|
||||
// letter (e.g. C:) or leading slash, and only forward slashes are
|
||||
// allowed. To create a directory instead of a file, add a trailing
|
||||
// slash to the name.
|
||||
// The file's contents must be written to the io.Writer before the next
|
||||
// call to Create, CreateHeader, CreateHeaderRaw, or Close.
|
||||
func (w *Writer) Create(name string) (io.Writer, error) {
|
||||
header := &FileHeader{
|
||||
Name: name,
|
||||
Method: Deflate,
|
||||
}
|
||||
return w.CreateHeader(header)
|
||||
}
|
||||
|
||||
// Copy will copy raw content from input file.
|
||||
// CHANGE: Optional file name cannot be specified any more due to stdlib api.
|
||||
func (w *Writer) Copy(src *File) error {
|
||||
header := src.FileHeader
|
||||
raw, err := src.OpenRaw()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
wr, err := w.CreateHeaderRaw(&header)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = io.Copy(wr, raw)
|
||||
return err
|
||||
}
|
||||
|
||||
// detectUTF8 reports whether s is a valid UTF-8 string, and whether the string
|
||||
// must be considered UTF-8 encoding (i.e., not compatible with CP-437, ASCII,
|
||||
// or any other common encoding).
|
||||
func detectUTF8(s string) (valid, require bool) {
|
||||
for i := 0; i < len(s); {
|
||||
r, size := utf8.DecodeRuneInString(s[i:])
|
||||
i += size
|
||||
// Officially, ZIP uses CP-437, but many readers use the system's
|
||||
// local character encoding. Most encoding are compatible with a large
|
||||
// subset of CP-437, which itself is ASCII-like.
|
||||
//
|
||||
// Forbid 0x7e and 0x5c since EUC-KR and Shift-JIS replace those
|
||||
// characters with localized currency and overline characters.
|
||||
if r < 0x20 || r > 0x7d || r == 0x5c {
|
||||
if !utf8.ValidRune(r) || (r == utf8.RuneError && size == 1) {
|
||||
return false, false
|
||||
}
|
||||
require = true
|
||||
}
|
||||
}
|
||||
return true, require
|
||||
}
|
||||
|
||||
// CreateHeader adds a file to the zip archive using the provided FileHeader
|
||||
// for the file metadata. Writer takes ownership of fh and may mutate
|
||||
// its fields. The caller must not modify fh after calling CreateHeader.
|
||||
//
|
||||
// This returns a Writer to which the file contents should be written.
|
||||
// The file's contents must be written to the io.Writer before the next
|
||||
// call to Create, Copy, CreateHeader, CreateHeaderRaw or Close.
|
||||
func (w *Writer) CreateHeader(fh *FileHeader) (io.Writer, error) {
|
||||
if w.last != nil && !w.last.Closed() {
|
||||
if err := w.last.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if len(w.dir) > 0 && w.dir[len(w.dir)-1].FileHeader == fh {
|
||||
// See https://golang.org/issue/11144 confusion.
|
||||
return nil, errors.New("archive/zip: invalid duplicate FileHeader")
|
||||
}
|
||||
|
||||
// The ZIP format has a sad state of affairs regarding character encoding.
|
||||
// Officially, the name and comment fields are supposed to be encoded
|
||||
// in CP-437 (which is mostly compatible with ASCII), unless the UTF-8
|
||||
// flag bit is set. However, there are several problems:
|
||||
//
|
||||
// * Many ZIP readers still do not support UTF-8.
|
||||
// * If the UTF-8 flag is cleared, several readers simply interpret the
|
||||
// name and comment fields as whatever the local system encoding is.
|
||||
//
|
||||
// In order to avoid breaking readers without UTF-8 support,
|
||||
// we avoid setting the UTF-8 flag if the strings are CP-437 compatible.
|
||||
// However, if the strings require multibyte UTF-8 encoding and is a
|
||||
// valid UTF-8 string, then we set the UTF-8 bit.
|
||||
//
|
||||
// For the case, where the user explicitly wants to specify the encoding
|
||||
// as UTF-8, they will need to set the flag bit themselves.
|
||||
utf8Valid1, utf8Require1 := detectUTF8(fh.Name)
|
||||
utf8Valid2, utf8Require2 := detectUTF8(fh.Comment)
|
||||
switch {
|
||||
case fh.NonUTF8:
|
||||
fh.Flags &^= 0x800
|
||||
case (utf8Require1 || utf8Require2) && (utf8Valid1 && utf8Valid2):
|
||||
fh.Flags |= 0x800
|
||||
}
|
||||
|
||||
fh.CreatorVersion = fh.CreatorVersion&0xff00 | zipVersion20 // preserve compatibility byte
|
||||
fh.ReaderVersion = zipVersion20
|
||||
|
||||
// If Modified is set, this takes precedence over MS-DOS timestamp fields.
|
||||
if !fh.Modified.IsZero() {
|
||||
// Contrary to the FileHeader.SetModTime method, we intentionally
|
||||
// do not convert to UTC, because we assume the user intends to encode
|
||||
// the date using the specified timezone. A user may want this control
|
||||
// because many legacy ZIP readers interpret the timestamp according
|
||||
// to the local timezone.
|
||||
//
|
||||
// The timezone is only non-UTC if a user directly sets the Modified
|
||||
// field directly themselves. All other approaches sets UTC.
|
||||
fh.ModifiedDate, fh.ModifiedTime = timeToMsDosTime(fh.Modified)
|
||||
|
||||
// Use "extended timestamp" format since this is what Info-ZIP uses.
|
||||
// Nearly every major ZIP implementation uses a different format,
|
||||
// but at least most seem to be able to understand the other formats.
|
||||
//
|
||||
// This format happens to be identical for both local and central header
|
||||
// if modification time is the only timestamp being encoded.
|
||||
var mbuf [9]byte // 2*SizeOf(uint16) + SizeOf(uint8) + SizeOf(uint32)
|
||||
mt := uint32(fh.Modified.Unix())
|
||||
eb := writeBuf(mbuf[:])
|
||||
eb.uint16(extTimeExtraID)
|
||||
eb.uint16(5) // Size: SizeOf(uint8) + SizeOf(uint32)
|
||||
eb.uint8(1) // Flags: ModTime
|
||||
eb.uint32(mt) // ModTime
|
||||
fh.Extra = append(fh.Extra, mbuf[:]...)
|
||||
}
|
||||
|
||||
var ow io.Writer
|
||||
h := &header{
|
||||
FileHeader: fh,
|
||||
offset: uint64(w.cw.count),
|
||||
}
|
||||
|
||||
if strings.HasSuffix(fh.Name, "/") {
|
||||
// Set the compression method to Store to ensure data length is truly zero,
|
||||
// which the writeHeader method always encodes for the size fields.
|
||||
// This is necessary as most compression formats have non-zero lengths
|
||||
// even when compressing an empty string.
|
||||
fh.Method = Store
|
||||
fh.Flags &^= 0x8 // we will not write a data descriptor
|
||||
|
||||
// Explicitly clear sizes as they have no meaning for directories.
|
||||
fh.CompressedSize = 0
|
||||
fh.CompressedSize64 = 0
|
||||
fh.UncompressedSize = 0
|
||||
fh.UncompressedSize64 = 0
|
||||
|
||||
ow = dirWriter{}
|
||||
w.last = nil
|
||||
} else {
|
||||
fh.Flags |= 0x8 // we will write a data descriptor
|
||||
|
||||
fw := &fileWriter{
|
||||
zipw: w.cw,
|
||||
compCount: &countWriter{w: w.cw},
|
||||
crc32: crc32.NewIEEE(),
|
||||
}
|
||||
comp := w.compressor(fh.Method)
|
||||
if comp == nil {
|
||||
return nil, ErrAlgorithm
|
||||
}
|
||||
var err error
|
||||
fw.comp, err = comp(fw.compCount)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fw.rawCount = &countWriter{w: fw.comp}
|
||||
fw.header = h
|
||||
ow = fw
|
||||
w.last = fw
|
||||
}
|
||||
w.dir = append(w.dir, h)
|
||||
if err := writeHeader(w.cw, fh); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// If we're creating a directory, fw is nil.
|
||||
return ow, nil
|
||||
}
|
||||
|
||||
// Deprecated: CreateHeaderRaw is replaced by CreateRaw (stdlib name).
|
||||
func (w *Writer) CreateHeaderRaw(fh *FileHeader) (io.Writer, error) {
|
||||
return w.CreateRaw(fh)
|
||||
}
|
||||
|
||||
// CreateRaw adds a file to the zip archive using the provided FileHeader
|
||||
// for the file metadata. Writer takes ownership of fh and may mutate
|
||||
// its fields. The caller must not modify fh after calling CreateHeaderRaw.
|
||||
//
|
||||
// This returns a Writer to which the compressed file contents should be written.
|
||||
// The file's contents must be written to the io.Writer before the next
|
||||
// call to Create, Copy, CreateHeader, CreateRaw or Close.
|
||||
//
|
||||
// Using this requires knowledge of populating the FileHeader correctly (the
|
||||
// UncompressedSize64 and CRC32 fields should be set and valid for the contents
|
||||
// written). For copying from an existing zip file, the Copy() function is
|
||||
// recommended.
|
||||
func (w *Writer) CreateRaw(fh *FileHeader) (io.Writer, error) {
|
||||
if w.last != nil && !w.last.Closed() {
|
||||
if err := w.last.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if len(w.dir) > 0 && w.dir[len(w.dir)-1].FileHeader == fh {
|
||||
// See https://golang.org/issue/11144 confusion.
|
||||
return nil, errors.New("archive/zip: invalid duplicate FileHeader")
|
||||
}
|
||||
|
||||
// The ZIP format has a sad state of affairs regarding character encoding.
|
||||
// Officially, the name and comment fields are supposed to be encoded
|
||||
// in CP-437 (which is mostly compatible with ASCII), unless the UTF-8
|
||||
// flag bit is set. However, there are several problems:
|
||||
//
|
||||
// * Many ZIP readers still do not support UTF-8.
|
||||
// * If the UTF-8 flag is cleared, several readers simply interpret the
|
||||
// name and comment fields as whatever the local system encoding is.
|
||||
//
|
||||
// In order to avoid breaking readers without UTF-8 support,
|
||||
// we avoid setting the UTF-8 flag if the strings are CP-437 compatible.
|
||||
// However, if the strings require multibyte UTF-8 encoding and is a
|
||||
// valid UTF-8 string, then we set the UTF-8 bit.
|
||||
//
|
||||
// For the case, where the user explicitly wants to specify the encoding
|
||||
// as UTF-8, they will need to set the flag bit themselves.
|
||||
utf8Valid1, utf8Require1 := detectUTF8(fh.Name)
|
||||
utf8Valid2, utf8Require2 := detectUTF8(fh.Comment)
|
||||
switch {
|
||||
case fh.NonUTF8:
|
||||
fh.Flags &^= 0x800
|
||||
case (utf8Require1 || utf8Require2) && (utf8Valid1 && utf8Valid2):
|
||||
fh.Flags |= 0x800
|
||||
}
|
||||
|
||||
fh.CreatorVersion = fh.CreatorVersion&0xff00 | zipVersion20 // preserve compatibility byte
|
||||
fh.ReaderVersion = zipVersion20
|
||||
|
||||
// If Modified is set, this takes precedence over MS-DOS timestamp fields.
|
||||
if !fh.Modified.IsZero() {
|
||||
// Contrary to the FileHeader.SetModTime method, we intentionally
|
||||
// do not convert to UTC, because we assume the user intends to encode
|
||||
// the date using the specified timezone. A user may want this control
|
||||
// because many legacy ZIP readers interpret the timestamp according
|
||||
// to the local timezone.
|
||||
//
|
||||
// The timezone is only non-UTC if a user directly sets the Modified
|
||||
// field directly themselves. All other approaches sets UTC.
|
||||
fh.ModifiedDate, fh.ModifiedTime = timeToMsDosTime(fh.Modified)
|
||||
|
||||
// Use "extended timestamp" format since this is what Info-ZIP uses.
|
||||
// Nearly every major ZIP implementation uses a different format,
|
||||
// but at least most seem to be able to understand the other formats.
|
||||
//
|
||||
// This format happens to be identical for both local and central header
|
||||
// if modification time is the only timestamp being encoded.
|
||||
var mbuf [9]byte // 2*SizeOf(uint16) + SizeOf(uint8) + SizeOf(uint32)
|
||||
mt := uint32(fh.Modified.Unix())
|
||||
eb := writeBuf(mbuf[:])
|
||||
eb.uint16(extTimeExtraID)
|
||||
eb.uint16(5) // Size: SizeOf(uint8) + SizeOf(uint32)
|
||||
eb.uint8(1) // Flags: ModTime
|
||||
eb.uint32(mt) // ModTime
|
||||
fh.Extra = append(fh.Extra, mbuf[:]...)
|
||||
}
|
||||
|
||||
var ow io.Writer
|
||||
h := &header{
|
||||
FileHeader: fh,
|
||||
offset: uint64(w.cw.count),
|
||||
}
|
||||
|
||||
if strings.HasSuffix(fh.Name, "/") {
|
||||
// Set the compression method to Store to ensure data length is truly zero,
|
||||
// which the writeHeader method always encodes for the size fields.
|
||||
// This is necessary as most compression formats have non-zero lengths
|
||||
// even when compressing an empty string.
|
||||
fh.Method = Store
|
||||
fh.Flags &^= 0x8 // we will not write a data descriptor
|
||||
|
||||
// Explicitly clear sizes as they have no meaning for directories.
|
||||
fh.CompressedSize = 0
|
||||
fh.CompressedSize64 = 0
|
||||
fh.UncompressedSize = 0
|
||||
fh.UncompressedSize64 = 0
|
||||
|
||||
ow = dirWriter{}
|
||||
w.last = nil
|
||||
} else {
|
||||
fh.Flags |= 0x8 // we will write a data descriptor
|
||||
|
||||
fw := &rawWriter{
|
||||
header: h,
|
||||
zipw: w.cw,
|
||||
rawCount: &countWriter{w: w.cw},
|
||||
}
|
||||
ow = fw
|
||||
w.last = fw
|
||||
}
|
||||
w.dir = append(w.dir, h)
|
||||
if err := writeHeader(w.cw, fh); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// If we're creating a directory, fw is nil.
|
||||
return ow, nil
|
||||
}
|
||||
|
||||
func writeHeader(w io.Writer, h *FileHeader) error {
|
||||
const maxUint16 = 1<<16 - 1
|
||||
if len(h.Name) > maxUint16 {
|
||||
return errLongName
|
||||
}
|
||||
if len(h.Extra) > maxUint16 {
|
||||
return errLongExtra
|
||||
}
|
||||
|
||||
var buf [fileHeaderLen]byte
|
||||
b := writeBuf(buf[:])
|
||||
b.uint32(uint32(fileHeaderSignature))
|
||||
b.uint16(h.ReaderVersion)
|
||||
b.uint16(h.Flags)
|
||||
b.uint16(h.Method)
|
||||
b.uint16(h.ModifiedTime)
|
||||
b.uint16(h.ModifiedDate)
|
||||
b.uint32(0) // since we are writing a data descriptor crc32,
|
||||
b.uint32(0) // compressed size,
|
||||
b.uint32(0) // and uncompressed size should be zero
|
||||
b.uint16(uint16(len(h.Name)))
|
||||
b.uint16(uint16(len(h.Extra)))
|
||||
if _, err := w.Write(buf[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.WriteString(w, h.Name); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := w.Write(h.Extra)
|
||||
return err
|
||||
}
|
||||
|
||||
// RegisterCompressor registers or overrides a custom compressor for a specific
|
||||
// method ID. If a compressor for a given method is not found, Writer will
|
||||
// default to looking up the compressor at the package level.
|
||||
func (w *Writer) RegisterCompressor(method uint16, comp Compressor) {
|
||||
if w.compressors == nil {
|
||||
w.compressors = make(map[uint16]Compressor)
|
||||
}
|
||||
w.compressors[method] = comp
|
||||
}
|
||||
|
||||
func (w *Writer) compressor(method uint16) Compressor {
|
||||
comp := w.compressors[method]
|
||||
if comp == nil {
|
||||
comp = compressor(method)
|
||||
}
|
||||
return comp
|
||||
}
|
||||
|
||||
type dirWriter struct{}
|
||||
|
||||
func (dirWriter) Write(b []byte) (int, error) {
|
||||
if len(b) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
return 0, errors.New("zip: write to directory")
|
||||
}
|
||||
|
||||
type fileWriter struct {
|
||||
*header
|
||||
zipw io.Writer
|
||||
rawCount *countWriter
|
||||
comp io.WriteCloser
|
||||
compCount *countWriter
|
||||
crc32 hash.Hash32
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (w *fileWriter) Write(p []byte) (int, error) {
|
||||
if w.closed {
|
||||
return 0, errors.New("zip: write to closed file")
|
||||
}
|
||||
w.crc32.Write(p)
|
||||
return w.rawCount.Write(p)
|
||||
}
|
||||
|
||||
func (w *fileWriter) Closed() bool {
|
||||
return w.closed
|
||||
}
|
||||
|
||||
func (w *fileWriter) Close() error {
|
||||
if w.closed {
|
||||
return errors.New("zip: file closed twice")
|
||||
}
|
||||
w.closed = true
|
||||
if err := w.comp.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// update FileHeader
|
||||
fh := w.header.FileHeader
|
||||
fh.CRC32 = w.crc32.Sum32()
|
||||
fh.CompressedSize64 = uint64(w.compCount.count)
|
||||
fh.UncompressedSize64 = uint64(w.rawCount.count)
|
||||
|
||||
if fh.isZip64() {
|
||||
fh.CompressedSize = uint32max
|
||||
fh.UncompressedSize = uint32max
|
||||
fh.ReaderVersion = zipVersion45 // requires 4.5 - File uses ZIP64 format extensions
|
||||
} else {
|
||||
fh.CompressedSize = uint32(fh.CompressedSize64)
|
||||
fh.UncompressedSize = uint32(fh.UncompressedSize64)
|
||||
}
|
||||
|
||||
// Write data descriptor. This is more complicated than one would
|
||||
// think, see e.g. comments in zipfile.c:putextended() and
|
||||
// http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=7073588.
|
||||
// The approach here is to write 8 byte sizes if needed without
|
||||
// adding a zip64 extra in the local header (too late anyway).
|
||||
var buf []byte
|
||||
if fh.isZip64() {
|
||||
buf = make([]byte, dataDescriptor64Len)
|
||||
} else {
|
||||
buf = make([]byte, dataDescriptorLen)
|
||||
}
|
||||
b := writeBuf(buf)
|
||||
b.uint32(dataDescriptorSignature) // de-facto standard, required by OS X
|
||||
b.uint32(fh.CRC32)
|
||||
if fh.isZip64() {
|
||||
b.uint64(fh.CompressedSize64)
|
||||
b.uint64(fh.UncompressedSize64)
|
||||
} else {
|
||||
b.uint32(fh.CompressedSize)
|
||||
b.uint32(fh.UncompressedSize)
|
||||
}
|
||||
_, err := w.zipw.Write(buf)
|
||||
return err
|
||||
}
|
||||
|
||||
type rawWriter struct {
|
||||
*header
|
||||
zipw io.Writer
|
||||
rawCount *countWriter
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (w *rawWriter) Write(p []byte) (int, error) {
|
||||
if w.closed {
|
||||
return 0, errors.New("zip: write to closed file")
|
||||
}
|
||||
return w.rawCount.Write(p)
|
||||
}
|
||||
|
||||
func (w *rawWriter) Closed() bool {
|
||||
return w.closed
|
||||
}
|
||||
|
||||
func (w *rawWriter) Close() error {
|
||||
if w.closed {
|
||||
return errors.New("zip: file closed twice")
|
||||
}
|
||||
w.closed = true
|
||||
fh := w.FileHeader
|
||||
fh.CompressedSize64 = uint64(w.rawCount.count)
|
||||
|
||||
if fh.isZip64() {
|
||||
fh.CompressedSize = uint32max
|
||||
fh.UncompressedSize = uint32max
|
||||
fh.ReaderVersion = zipVersion45 // requires 4.5 - File uses ZIP64 format extensions
|
||||
} else {
|
||||
fh.CompressedSize = uint32(fh.CompressedSize64)
|
||||
fh.UncompressedSize = uint32(fh.UncompressedSize64)
|
||||
}
|
||||
|
||||
// Write data descriptor. This is more complicated than one would
|
||||
// think, see e.g. comments in zipfile.c:putextended() and
|
||||
// http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=7073588.
|
||||
// The approach here is to write 8 byte sizes if needed without
|
||||
// adding a zip64 extra in the local header (too late anyway).
|
||||
var buf []byte
|
||||
if fh.isZip64() {
|
||||
buf = make([]byte, dataDescriptor64Len)
|
||||
} else {
|
||||
buf = make([]byte, dataDescriptorLen)
|
||||
}
|
||||
b := writeBuf(buf)
|
||||
b.uint32(dataDescriptorSignature) // de-facto standard, required by OS X
|
||||
b.uint32(fh.CRC32)
|
||||
if fh.isZip64() {
|
||||
b.uint64(fh.CompressedSize64)
|
||||
b.uint64(fh.UncompressedSize64)
|
||||
} else {
|
||||
b.uint32(fh.CompressedSize)
|
||||
b.uint32(fh.UncompressedSize)
|
||||
}
|
||||
_, err := w.zipw.Write(buf)
|
||||
return err
|
||||
}
|
||||
|
||||
type countWriter struct {
|
||||
w io.Writer
|
||||
count int64
|
||||
}
|
||||
|
||||
func (w *countWriter) Write(p []byte) (int, error) {
|
||||
n, err := w.w.Write(p)
|
||||
w.count += int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
type nopCloser struct {
|
||||
io.Writer
|
||||
}
|
||||
|
||||
func (w nopCloser) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type writeBuf []byte
|
||||
|
||||
func (b *writeBuf) uint8(v uint8) {
|
||||
(*b)[0] = v
|
||||
*b = (*b)[1:]
|
||||
}
|
||||
|
||||
func (b *writeBuf) uint16(v uint16) {
|
||||
binary.LittleEndian.PutUint16(*b, v)
|
||||
*b = (*b)[2:]
|
||||
}
|
||||
|
||||
func (b *writeBuf) uint32(v uint32) {
|
||||
binary.LittleEndian.PutUint32(*b, v)
|
||||
*b = (*b)[4:]
|
||||
}
|
||||
|
||||
func (b *writeBuf) uint64(v uint64) {
|
||||
binary.LittleEndian.PutUint64(*b, v)
|
||||
*b = (*b)[8:]
|
||||
}
|
||||
Ссылка в новой задаче
Block a user