diff --git a/.gitignore b/.gitignore index ec975e873c..366d1c3744 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,6 @@ project.lock.json .build/ .testPublish/ global.json + +# Dependencies from pre-requisite builds +.deps/ diff --git a/MetaPackages.sln b/MetaPackages.sln index de2568595f..95ab066bb5 100644 --- a/MetaPackages.sln +++ b/MetaPackages.sln @@ -45,6 +45,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "TestArtifacts", "TestArtifa test\TestArtifacts\testCert.pfx = test\TestArtifacts\testCert.pfx EndProjectSection EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "dotnet-archive", "src\dotnet-archive\dotnet-archive.csproj", "{AE4216BF-D471-471B-82F3-6B6D004F7D17}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.DotNet.Archive", "src\Microsoft.DotNet.Archive\Microsoft.DotNet.Archive.csproj", "{302400A0-98BB-4C04-88D4-C32DC2D4B945}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -87,6 +91,14 @@ Global {79CF58CE-B020-45D8-BDB5-2D8036BEAD14}.Debug|Any CPU.Build.0 = Debug|Any CPU {79CF58CE-B020-45D8-BDB5-2D8036BEAD14}.Release|Any CPU.ActiveCfg = Release|Any CPU {79CF58CE-B020-45D8-BDB5-2D8036BEAD14}.Release|Any CPU.Build.0 = Release|Any CPU + {AE4216BF-D471-471B-82F3-6B6D004F7D17}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AE4216BF-D471-471B-82F3-6B6D004F7D17}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AE4216BF-D471-471B-82F3-6B6D004F7D17}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AE4216BF-D471-471B-82F3-6B6D004F7D17}.Release|Any CPU.Build.0 = Release|Any CPU + {302400A0-98BB-4C04-88D4-C32DC2D4B945}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {302400A0-98BB-4C04-88D4-C32DC2D4B945}.Debug|Any CPU.Build.0 = Debug|Any CPU + {302400A0-98BB-4C04-88D4-C32DC2D4B945}.Release|Any CPU.ActiveCfg = Release|Any CPU + {302400A0-98BB-4C04-88D4-C32DC2D4B945}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -104,5 +116,7 @@ Global {401C741B-6C7C-4E08-9F09-C3D43D22C0DE} = {EC22261D-0DE1-47DE-8F7C-072675D6F5B4} {79CF58CE-B020-45D8-BDB5-2D8036BEAD14} = {EC22261D-0DE1-47DE-8F7C-072675D6F5B4} {9BBA7A0A-109A-4AC8-B6EF-A52EA7CF1D90} = {9E49B5B9-9E72-42FB-B684-90CA1B1BCF9C} + {AE4216BF-D471-471B-82F3-6B6D004F7D17} = {ED834E68-51C3-4ADE-ACC8-6BA6D4207C09} + {302400A0-98BB-4C04-88D4-C32DC2D4B945} = {ED834E68-51C3-4ADE-ACC8-6BA6D4207C09} EndGlobalSection EndGlobal diff --git a/build/BuildArchive.proj b/build/BuildArchive.proj new file mode 100644 index 0000000000..eadad66b41 --- /dev/null +++ b/build/BuildArchive.proj @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/build/common.props b/build/common.props index b8f34e09fe..fcf40a899f 100644 --- a/build/common.props +++ b/build/common.props @@ -10,11 +10,15 @@ ..\..\build\Key.snk true true - $(VersionSuffix)-$(BuildNumber) + $(VersionSuffix) + + + $(VersionSuffix) + $(VersionSuffix)-$(BuildNumber) - 2.0.0-$(VersionSuffix) - 1.0.0-$(VersionSuffix) + 2.0.0-$(BuildVersionSuffix) + 1.0.0-$(BuildVersionSuffix) diff --git a/build/repo.props b/build/repo.props index c7b2520845..0b28f5ecaa 100644 --- a/build/repo.props +++ b/build/repo.props @@ -1,5 +1,8 @@ + + + - \ No newline at end of file + diff --git a/build/repo.targets b/build/repo.targets index 787bead341..09a0f36d7d 100644 --- a/build/repo.targets +++ b/build/repo.targets @@ -1,9 +1,17 @@ - + + $(RepositoryRoot)src\Microsoft.AspNetCore.RuntimeStore\ $(MetaPackagePath)Microsoft.AspNetCore.RuntimeStore.csproj + + + $(RepositoryRoot)src\Archive.AspNetCore.All\ + $(FallbackArchiveDir)Archive.AspNetCore.All.csproj + $(RepositoryRoot)src\dotnet-archive\dotnet-archive.csproj + $(RepositoryRoot)src\dotnet-archive\bin\$(Configuration)\netcoreapp2.0\dotnet-archive.dll + $(MetaPackagePath)bin\work\ $(MetaPackagePath)bin\packageCache\ $(MetaPackagePath)bin\deps\ @@ -20,6 +28,10 @@ $(CompileDependsOn); BuildPackageCache + + $(CompileDependsOn); + BuildFallbackArchive + @@ -88,6 +100,73 @@ + + + + + + + + + + + $(FallbackArchiveDir)obj/$(OutputPackageName) + $(RepositoryRoot)artifacts\$(OutputPackageName).lzma + $(FallbackArchiveDir)\obj\$(OutputPackageName).NuGet.config + + + + + + + + + + + + + + + + + + + + + + + + + $(COHERENCE_SIGNED_DROP_ROOT) + \\aspnetci\Drops\Coherence-Signed\dev + + $(TIMESTAMP_FREE_VERSION) + final + + $(VersionPrefix)-$(VersionSuffix)-$(NoTimestampSuffix) + $(VersionPrefix)-$(VersionSuffix)-$(BuildNumber) + + $(CoherenceSignedRoot)\$(BuildNumber)\Signed\Packages-NoTimeStamp + $(CoherenceSignedRoot)\$(BuildNumber)\Signed\Packages + + + + + + + + + + + $(PublishShare)\fallbackArchives + + + + + @@ -108,4 +187,4 @@ - \ No newline at end of file + diff --git a/src/Archive.AspNetCore.All/Archive.AspNetCore.All.csproj b/src/Archive.AspNetCore.All/Archive.AspNetCore.All.csproj new file mode 100644 index 0000000000..5d1de6a2bb --- /dev/null +++ b/src/Archive.AspNetCore.All/Archive.AspNetCore.All.csproj @@ -0,0 +1,31 @@ + + + + netcoreapp2.0 + $(PackageTargetFallback);portable-net45+win8+wp8+wpa81; + false + + + + + + + + + + + + + + diff --git a/src/Microsoft.AspNetCore.RuntimeStore/Microsoft.AspNetCore.RuntimeStore.csproj b/src/Microsoft.AspNetCore.RuntimeStore/Microsoft.AspNetCore.RuntimeStore.csproj index 27d7905342..0fbcd07cd7 100644 --- a/src/Microsoft.AspNetCore.RuntimeStore/Microsoft.AspNetCore.RuntimeStore.csproj +++ b/src/Microsoft.AspNetCore.RuntimeStore/Microsoft.AspNetCore.RuntimeStore.csproj @@ -19,4 +19,4 @@ - \ No newline at end of file + diff --git a/src/Microsoft.DotNet.Archive/CompressionUtility.cs b/src/Microsoft.DotNet.Archive/CompressionUtility.cs new file mode 100644 index 0000000000..0b4e937dc6 --- /dev/null +++ b/src/Microsoft.DotNet.Archive/CompressionUtility.cs @@ -0,0 +1,107 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using SevenZip; +using System; +using System.IO; + +namespace Microsoft.DotNet.Archive +{ + internal static class CompressionUtility + { + enum MeasureBy + { + Input, + Output + } + + private class LzmaProgress : ICodeProgress + { + private IProgress progress; + private long totalSize; + private string phase; + private MeasureBy measureBy; + + public LzmaProgress(IProgress progress, string phase, long totalSize, MeasureBy measureBy) + { + this.progress = progress; + this.totalSize = totalSize; + this.phase = phase; + this.measureBy = measureBy; + } + + public void SetProgress(long inSize, long outSize) + { + progress.Report(phase, measureBy == MeasureBy.Input ? inSize : outSize, totalSize); + } + } + + public static void Compress(Stream inStream, Stream outStream, IProgress progress) + { + SevenZip.Compression.LZMA.Encoder encoder = new SevenZip.Compression.LZMA.Encoder(); + + CoderPropID[] propIDs = + { + CoderPropID.DictionarySize, + CoderPropID.PosStateBits, + CoderPropID.LitContextBits, + CoderPropID.LitPosBits, + CoderPropID.Algorithm, + CoderPropID.NumFastBytes, + CoderPropID.MatchFinder, + CoderPropID.EndMarker + }; + object[] properties = + { + (Int32)(1 << 26), + (Int32)(1), + (Int32)(8), + (Int32)(0), + (Int32)(2), + (Int32)(96), + "bt4", + false + }; + + encoder.SetCoderProperties(propIDs, properties); + encoder.WriteCoderProperties(outStream); + + Int64 inSize = inStream.Length; + for (int i = 0; i < 8; i++) + { + outStream.WriteByte((Byte)(inSize >> (8 * i))); + } + + var lzmaProgress = new LzmaProgress(progress, "Compressing", inSize, MeasureBy.Input); + lzmaProgress.SetProgress(0, 0); + encoder.Code(inStream, outStream, -1, -1, lzmaProgress); + lzmaProgress.SetProgress(inSize, outStream.Length); + } + + public static void Decompress(Stream inStream, Stream outStream, IProgress progress) + { + byte[] properties = new byte[5]; + + if (inStream.Read(properties, 0, 5) != 5) + throw (new Exception("input .lzma is too short")); + + SevenZip.Compression.LZMA.Decoder decoder = new SevenZip.Compression.LZMA.Decoder(); + decoder.SetDecoderProperties(properties); + + long outSize = 0; + for (int i = 0; i < 8; i++) + { + int v = inStream.ReadByte(); + if (v < 0) + throw (new Exception("Can't Read 1")); + outSize |= ((long)(byte)v) << (8 * i); + } + + long compressedSize = inStream.Length - inStream.Position; + var lzmaProgress = new LzmaProgress(progress, "Decompressing", outSize, MeasureBy.Output); + lzmaProgress.SetProgress(0, 0); + decoder.Code(inStream, outStream, compressedSize, outSize, lzmaProgress); + lzmaProgress.SetProgress(inStream.Length, outSize); + } + } +} diff --git a/src/Microsoft.DotNet.Archive/ConsoleProgressReport.cs b/src/Microsoft.DotNet.Archive/ConsoleProgressReport.cs new file mode 100644 index 0000000000..e8f6cd0dff --- /dev/null +++ b/src/Microsoft.DotNet.Archive/ConsoleProgressReport.cs @@ -0,0 +1,59 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Diagnostics; + +namespace Microsoft.DotNet.Archive +{ + public class ConsoleProgressReport : IProgress + { + private string _currentPhase; + private int _lastLineLength = 0; + private double _lastProgress = -1; + private Stopwatch _stopwatch; + private object _stateLock = new object(); + + public void Report(ProgressReport value) + { + long progress = (long)(100 * ((double)value.Ticks / value.Total)); + + if (progress == _lastProgress && value.Phase == _currentPhase) + { + return; + } + _lastProgress = progress; + + lock (_stateLock) + { + string line = $"{value.Phase} {progress}%"; + if (value.Phase == _currentPhase) + { + if (Console.IsOutputRedirected) + { + Console.Write($"...{progress}%"); + } + else + { + Console.Write(new string('\b', _lastLineLength)); + Console.Write(line); + } + + _lastLineLength = line.Length; + + if (progress == 100) + { + Console.WriteLine($" {_stopwatch.ElapsedMilliseconds} ms"); + } + } + else + { + Console.Write(line); + _currentPhase = value.Phase; + _lastLineLength = line.Length; + _stopwatch = Stopwatch.StartNew(); + } + } + } + } +} diff --git a/src/Microsoft.DotNet.Archive/IndexedArchive.cs b/src/Microsoft.DotNet.Archive/IndexedArchive.cs new file mode 100644 index 0000000000..bb4c9bbd14 --- /dev/null +++ b/src/Microsoft.DotNet.Archive/IndexedArchive.cs @@ -0,0 +1,538 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Threading; + +namespace Microsoft.DotNet.Archive +{ + public class IndexedArchive : IDisposable + { + private class DestinationFileInfo + { + public DestinationFileInfo(string destinationPath, string hash) + { + DestinationPath = destinationPath; + Hash = hash; + } + + public string DestinationPath { get; } + public string Hash { get; } + } + + private class ArchiveSource + { + public ArchiveSource(string sourceArchive, string sourceFile, string archivePath, string hash, long size) + { + SourceArchive = sourceArchive; + SourceFile = sourceFile; + ArchivePath = archivePath; + Hash = hash; + Size = size; + } + + public string SourceArchive { get; set; } + public string SourceFile { get; set; } + public string ArchivePath { get; } + public string Hash { get; } + public string FileName { get { return Path.GetFileNameWithoutExtension(ArchivePath); } } + public string Extension { get { return Path.GetExtension(ArchivePath); } } + public long Size { get; } + + public void CopyTo(Stream destination) + { + if (!String.IsNullOrEmpty(SourceArchive)) + { + using (var zip = new ZipArchive(File.OpenRead(SourceArchive), ZipArchiveMode.Read)) + using (var sourceStream = zip.GetEntry(SourceFile)?.Open()) + { + if (sourceStream == null) + { + throw new Exception($"Couldn't find entry {SourceFile} in archive {SourceArchive}"); + } + + sourceStream.CopyTo(destination); + } + } + else + { + using (var sourceStream = File.OpenRead(SourceFile)) + { + sourceStream.CopyTo(destination); + } + } + } + } + + static string[] ZipExtensions = new[] { ".zip", ".nupkg" }; + static string IndexFileName = "index.txt"; + + // maps file hash to archve path + // $ prefix indicates that the file is not in the archive and path is a hash + private Dictionary _archiveFiles = new Dictionary(); + // maps file hash to external path + private Dictionary _externalFiles = new Dictionary(); + // lists all extracted files & hashes + private List _destFiles = new List(); + private bool _disposed = false; + private ThreadLocal _sha = new ThreadLocal(() => SHA256.Create()); + + public IndexedArchive() + { } + + private static Stream CreateTemporaryStream() + { + string temp = Path.GetTempPath(); + string tempFile = Path.Combine(temp, Guid.NewGuid().ToString()); + return File.Create(tempFile, 4096, FileOptions.DeleteOnClose); + } + + private static FileStream CreateTemporaryFileStream() + { + string temp = Path.GetTempPath(); + string tempFile = Path.Combine(temp, Guid.NewGuid().ToString()); + return new FileStream(tempFile, FileMode.Create, FileAccess.ReadWrite, FileShare.Read | FileShare.Delete, 4096, FileOptions.DeleteOnClose); + } + + public void Save(string archivePath, IProgress progress) + { + CheckDisposed(); + + using (var archiveStream = CreateTemporaryStream()) + { + using (var archive = new ZipArchive(archiveStream, ZipArchiveMode.Create, true)) + { + BuildArchive(archive, progress); + } // close archive + + archiveStream.Seek(0, SeekOrigin.Begin); + + using (var lzmaStream = File.Create(archivePath)) + { + CompressionUtility.Compress(archiveStream, lzmaStream, progress); + } + } // close archiveStream + } + + private void BuildArchive(ZipArchive archive, IProgress progress) + { + // write the file index + var indexEntry = archive.CreateEntry(IndexFileName, CompressionLevel.NoCompression); + + using (var stream = indexEntry.Open()) + using (var textWriter = new StreamWriter(stream)) + { + foreach (var entry in _destFiles) + { + var archiveFile = _archiveFiles[entry.Hash]; + string archivePath = _archiveFiles[entry.Hash].ArchivePath; + if (archiveFile.SourceFile == null) + { + archivePath = "$" + archivePath; + } + + textWriter.WriteLine($"{entry.DestinationPath}|{archivePath}"); + } + } + + // sort the files so that similar files are close together + var filesToArchive = _archiveFiles.Values.ToList(); + filesToArchive.Sort((f1, f2) => + { + // first sort by extension + var comp = String.Compare(f1.Extension, f2.Extension, StringComparison.OrdinalIgnoreCase); + + if (comp == 0) + { + // then sort by filename + comp = String.Compare(f1.FileName, f2.FileName, StringComparison.OrdinalIgnoreCase); + } + + if (comp == 0) + { + // sort by file size (helps differentiate ref/lib/facade) + comp = f1.Size.CompareTo(f2.Size); + } + + if (comp == 0) + { + // finally sort by full archive path so we have stable output + comp = String.Compare(f1.ArchivePath, f2.ArchivePath, StringComparison.OrdinalIgnoreCase); + } + + return comp; + }); + + int filesAdded = 0; + // add all the files + foreach (var fileToArchive in filesToArchive) + { + var entry = archive.CreateEntry(fileToArchive.ArchivePath, CompressionLevel.NoCompression); + using (var entryStream = entry.Open()) + { + fileToArchive.CopyTo(entryStream); + } + + progress.Report("Archiving files", ++filesAdded, filesToArchive.Count); + } + } + + private abstract class ExtractOperation + { + public ExtractOperation(string destinationPath) + { + DestinationPath = destinationPath; + } + + public string DestinationPath { get; } + public virtual void DoOperation() + { + string directory = Path.GetDirectoryName(DestinationPath); + + if (!Directory.Exists(directory)) + { + Directory.CreateDirectory(directory); + } + + Execute(); + } + protected abstract void Execute(); + } + + private class CopyOperation : ExtractOperation + { + public CopyOperation(ExtractSource source, string destinationPath) : base(destinationPath) + { + Source = source; + } + public ExtractSource Source { get; } + protected override void Execute() + { + if (Source.LocalPath != null) + { + File.Copy(Source.LocalPath, DestinationPath, true); + } + else + { + using (var destinationStream = File.Create(DestinationPath)) + { + Source.CopyToStream(destinationStream); + } + } + } + } + + private class ZipOperation : ExtractOperation + { + public ZipOperation(string destinationPath) : base(destinationPath) + { + } + + private List> entries = new List>(); + + public void AddEntry(string entryName, ExtractSource source) + { + entries.Add(Tuple.Create(entryName, source)); + } + + protected override void Execute() + { + using (var archiveStream = File.Create(DestinationPath)) + using (var archive = new ZipArchive(archiveStream, ZipArchiveMode.Create)) + { + foreach(var zipSource in entries) + { + var entry = archive.CreateEntry(zipSource.Item1, CompressionLevel.Optimal); + using (var entryStream = entry.Open()) + { + zipSource.Item2.CopyToStream(entryStream); + } + } + } + } + } + + private class ExtractSource + { + private string _entryName; + private readonly string _localPath; + private ThreadLocalZipArchive _archive; + + public ExtractSource(string sourceString, Dictionary externalFiles, ThreadLocalZipArchive archive) + { + if (sourceString[0] == '$') + { + var externalHash = sourceString.Substring(1); + if (!externalFiles.TryGetValue(externalHash, out _localPath)) + { + throw new Exception("Could not find external file with hash {externalHash}."); + } + } + else + { + _entryName = sourceString; + _archive = archive; + } + } + + public string LocalPath { get { return _localPath; } } + + public void CopyToStream(Stream destinationStream) + { + if (_localPath != null) + { + using (var sourceStream = File.OpenRead(_localPath)) + { + sourceStream.CopyTo(destinationStream); + } + } + else + { + using (var sourceStream = _archive.Archive.GetEntry(_entryName).Open()) + { + sourceStream.CopyTo(destinationStream); + } + } + + } + } + + private static char[] pipeSeperator = new[] { '|' }; + public void Extract(string compressedArchivePath, string outputDirectory, IProgress progress) + { + using (var archiveStream = CreateTemporaryFileStream()) + { + // decompress the LZMA stream + using (var lzmaStream = File.OpenRead(compressedArchivePath)) + { + CompressionUtility.Decompress(lzmaStream, archiveStream, progress); + } + + var archivePath = ((FileStream)archiveStream).Name; + + // reset the uncompressed stream + archiveStream.Seek(0, SeekOrigin.Begin); + + // read as a zip archive + using (var archive = new ZipArchive(archiveStream, ZipArchiveMode.Read)) + using (var tlArchive = new ThreadLocalZipArchive(archivePath, archive)) + { + List extractOperations = new List(); + Dictionary sourceCache = new Dictionary(); + + // process the index to determine all extraction operations + var indexEntry = archive.GetEntry(IndexFileName); + using (var indexReader = new StreamReader(indexEntry.Open())) + { + Dictionary zipOperations = new Dictionary(StringComparer.OrdinalIgnoreCase); + for (var line = indexReader.ReadLine(); line != null; line = indexReader.ReadLine()) + { + var lineParts = line.Split(pipeSeperator); + if (lineParts.Length != 2) + { + throw new Exception("Unexpected index line format, too many '|'s."); + } + + string target = lineParts[0]; + string source = lineParts[1]; + + ExtractSource extractSource; + if (!sourceCache.TryGetValue(source, out extractSource)) + { + sourceCache[source] = extractSource = new ExtractSource(source, _externalFiles, tlArchive); + } + + var zipSeperatorIndex = target.IndexOf("::", StringComparison.OrdinalIgnoreCase); + + if (zipSeperatorIndex != -1) + { + string zipRelativePath = target.Substring(0, zipSeperatorIndex); + string zipEntryName = target.Substring(zipSeperatorIndex + 2); + string destinationPath = Path.Combine(outputDirectory, zipRelativePath); + + // operations on a zip file will be sequential + ZipOperation currentZipOperation; + + if (!zipOperations.TryGetValue(destinationPath, out currentZipOperation)) + { + extractOperations.Add(currentZipOperation = new ZipOperation(destinationPath)); + zipOperations.Add(destinationPath, currentZipOperation); + } + currentZipOperation.AddEntry(zipEntryName, extractSource); + } + else + { + string destinationPath = Path.Combine(outputDirectory, target); + extractOperations.Add(new CopyOperation(extractSource, destinationPath)); + } + } + } + + int opsExecuted = 0; + // execute all operations + //foreach(var extractOperation in extractOperations) + extractOperations.AsParallel().ForAll(extractOperation => + { + extractOperation.DoOperation(); + progress.Report("Expanding", Interlocked.Increment(ref opsExecuted), extractOperations.Count); + }); + } + } + } + + public void AddExternalDirectory(string externalDirectory) + { + CheckDisposed(); + foreach (var externalFile in Directory.EnumerateFiles(externalDirectory, "*", SearchOption.AllDirectories)) + { + AddExternalFile(externalFile); + } + } + + public void AddExternalFile(string externalFile) + { + CheckDisposed(); + using (var fs = File.OpenRead(externalFile)) + { + string hash = GetHash(fs); + // $ prefix indicates that the file is not in the archive and path is relative to an external directory + _archiveFiles[hash] = new ArchiveSource(null, null, "$" + hash , hash, fs.Length); + _externalFiles[hash] = externalFile; + } + } + public void AddDirectory(string sourceDirectory, IProgress progress, string destinationDirectory = null) + { + var sourceFiles = Directory.EnumerateFiles(sourceDirectory, "*", SearchOption.AllDirectories).ToArray(); + int filesAdded = 0; + sourceFiles.AsParallel().ForAll(sourceFile => + { + // path relative to the destination/extracted directory to write the file + string destinationRelativePath = sourceFile.Substring(sourceDirectory.Length + 1); + + if (destinationDirectory != null) + { + destinationRelativePath = Path.Combine(destinationDirectory, destinationRelativePath); + } + + string extension = Path.GetExtension(sourceFile); + + if (ZipExtensions.Any(ze => ze.Equals(extension, StringComparison.OrdinalIgnoreCase))) + { + AddZip(sourceFile, destinationRelativePath); + } + else + { + AddFile(sourceFile, destinationRelativePath); + } + + progress.Report($"Adding {sourceDirectory}", Interlocked.Increment(ref filesAdded), sourceFiles.Length); + }); + } + + public void AddZip(string sourceZipFile, string destinationZipFile) + { + CheckDisposed(); + + using (var sourceArchive = new ZipArchive(File.OpenRead(sourceZipFile), ZipArchiveMode.Read)) + { + foreach(var entry in sourceArchive.Entries) + { + string hash = null; + long size = entry.Length; + string destinationPath = $"{destinationZipFile}::{entry.FullName}"; + using (var stream = entry.Open()) + { + hash = GetHash(stream); + } + + AddArchiveSource(sourceZipFile, entry.FullName, destinationPath, hash, size); + } + } + } + + public void AddFile(string sourceFilePath, string destinationPath) + { + CheckDisposed(); + + string hash; + long size; + // lifetime of this stream is managed by AddStream + using (var stream = File.Open(sourceFilePath, FileMode.Open)) + { + hash = GetHash(stream); + size = stream.Length; + } + + AddArchiveSource(null, sourceFilePath, destinationPath, hash, size); + } + + private void AddArchiveSource(string sourceArchive, string sourceFile, string destinationPath, string hash, long size) + { + lock (_archiveFiles) + { + _destFiles.Add(new DestinationFileInfo(destinationPath, hash)); + + // see if we already have this file in the archive/external + ArchiveSource existing = null; + if (_archiveFiles.TryGetValue(hash, out existing)) + { + // if we have raw source file, prefer that over a zipped source file + if (sourceArchive == null && existing.SourceArchive != null) + { + existing.SourceArchive = null; + existing.SourceFile = sourceFile; + } + } + else + { + var archivePath = Path.Combine(hash, Path.GetFileName(destinationPath)); + + _archiveFiles.Add(hash, new ArchiveSource(sourceArchive, sourceFile, archivePath, hash, size)); + } + } + } + + public string GetHash(Stream stream) + { + var hashBytes = _sha.Value.ComputeHash(stream); + + return GetHashString(hashBytes); + } + + private static string GetHashString(byte[] hashBytes) + { + StringBuilder builder = new StringBuilder(hashBytes.Length * 2); + foreach (var b in hashBytes) + { + builder.AppendFormat("{0:x2}", b); + } + return builder.ToString(); + } + + public void Dispose() + { + if (!_disposed) + { + if (_sha != null) + { + _sha.Dispose(); + _sha = null; + } + } + } + + private void CheckDisposed() + { + if (_disposed) + { + throw new ObjectDisposedException(nameof(IndexedArchive)); + } + } + } +} diff --git a/src/Microsoft.DotNet.Archive/LZMA/Common/CRC.cs b/src/Microsoft.DotNet.Archive/LZMA/Common/CRC.cs new file mode 100644 index 0000000000..5d38bf911b --- /dev/null +++ b/src/Microsoft.DotNet.Archive/LZMA/Common/CRC.cs @@ -0,0 +1,58 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +// Common/CRC.cs + +namespace SevenZip +{ + class CRC + { + public static readonly uint[] Table; + + static CRC() + { + Table = new uint[256]; + const uint kPoly = 0xEDB88320; + for (uint i = 0; i < 256; i++) + { + uint r = i; + for (int j = 0; j < 8; j++) + if ((r & 1) != 0) + r = (r >> 1) ^ kPoly; + else + r >>= 1; + Table[i] = r; + } + } + + uint _value = 0xFFFFFFFF; + + public void Init() { _value = 0xFFFFFFFF; } + + public void UpdateByte(byte b) + { + _value = Table[(((byte)(_value)) ^ b)] ^ (_value >> 8); + } + + public void Update(byte[] data, uint offset, uint size) + { + for (uint i = 0; i < size; i++) + _value = Table[(((byte)(_value)) ^ data[offset + i])] ^ (_value >> 8); + } + + public uint GetDigest() { return _value ^ 0xFFFFFFFF; } + + static uint CalculateDigest(byte[] data, uint offset, uint size) + { + CRC crc = new CRC(); + // crc.Init(); + crc.Update(data, offset, size); + return crc.GetDigest(); + } + + static bool VerifyDigest(uint digest, byte[] data, uint offset, uint size) + { + return (CalculateDigest(data, offset, size) == digest); + } + } +} diff --git a/src/Microsoft.DotNet.Archive/LZMA/Common/InBuffer.cs b/src/Microsoft.DotNet.Archive/LZMA/Common/InBuffer.cs new file mode 100644 index 0000000000..a26bf4a29b --- /dev/null +++ b/src/Microsoft.DotNet.Archive/LZMA/Common/InBuffer.cs @@ -0,0 +1,75 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +// InBuffer.cs + +namespace SevenZip.Buffer +{ + public class InBuffer + { + byte[] m_Buffer; + uint m_Pos; + uint m_Limit; + uint m_BufferSize; + System.IO.Stream m_Stream; + bool m_StreamWasExhausted; + ulong m_ProcessedSize; + + public InBuffer(uint bufferSize) + { + m_Buffer = new byte[bufferSize]; + m_BufferSize = bufferSize; + } + + public void Init(System.IO.Stream stream) + { + m_Stream = stream; + m_ProcessedSize = 0; + m_Limit = 0; + m_Pos = 0; + m_StreamWasExhausted = false; + } + + public bool ReadBlock() + { + if (m_StreamWasExhausted) + return false; + m_ProcessedSize += m_Pos; + int aNumProcessedBytes = m_Stream.Read(m_Buffer, 0, (int)m_BufferSize); + m_Pos = 0; + m_Limit = (uint)aNumProcessedBytes; + m_StreamWasExhausted = (aNumProcessedBytes == 0); + return (!m_StreamWasExhausted); + } + + + public void ReleaseStream() + { + // m_Stream.Close(); + m_Stream = null; + } + + public bool ReadByte(byte b) // check it + { + if (m_Pos >= m_Limit) + if (!ReadBlock()) + return false; + b = m_Buffer[m_Pos++]; + return true; + } + + public byte ReadByte() + { + // return (byte)m_Stream.ReadByte(); + if (m_Pos >= m_Limit) + if (!ReadBlock()) + return 0xFF; + return m_Buffer[m_Pos++]; + } + + public ulong GetProcessedSize() + { + return m_ProcessedSize + m_Pos; + } + } +} diff --git a/src/Microsoft.DotNet.Archive/LZMA/Common/OutBuffer.cs b/src/Microsoft.DotNet.Archive/LZMA/Common/OutBuffer.cs new file mode 100644 index 0000000000..429bccfc91 --- /dev/null +++ b/src/Microsoft.DotNet.Archive/LZMA/Common/OutBuffer.cs @@ -0,0 +1,50 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +// OutBuffer.cs + +namespace SevenZip.Buffer +{ + public class OutBuffer + { + byte[] m_Buffer; + uint m_Pos; + uint m_BufferSize; + System.IO.Stream m_Stream; + ulong m_ProcessedSize; + + public OutBuffer(uint bufferSize) + { + m_Buffer = new byte[bufferSize]; + m_BufferSize = bufferSize; + } + + public void SetStream(System.IO.Stream stream) { m_Stream = stream; } + public void FlushStream() { m_Stream.Flush(); } + public void CloseStream() { m_Stream.Dispose(); } + public void ReleaseStream() { m_Stream = null; } + + public void Init() + { + m_ProcessedSize = 0; + m_Pos = 0; + } + + public void WriteByte(byte b) + { + m_Buffer[m_Pos++] = b; + if (m_Pos >= m_BufferSize) + FlushData(); + } + + public void FlushData() + { + if (m_Pos == 0) + return; + m_Stream.Write(m_Buffer, 0, (int)m_Pos); + m_Pos = 0; + } + + public ulong GetProcessedSize() { return m_ProcessedSize + m_Pos; } + } +} diff --git a/src/Microsoft.DotNet.Archive/LZMA/Compress/LZ/IMatchFinder.cs b/src/Microsoft.DotNet.Archive/LZMA/Compress/LZ/IMatchFinder.cs new file mode 100644 index 0000000000..2916aedb0f --- /dev/null +++ b/src/Microsoft.DotNet.Archive/LZMA/Compress/LZ/IMatchFinder.cs @@ -0,0 +1,27 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +// IMatchFinder.cs + +using System; + +namespace SevenZip.Compression.LZ +{ + interface IInWindowStream + { + void SetStream(System.IO.Stream inStream); + void Init(); + void ReleaseStream(); + Byte GetIndexByte(Int32 index); + UInt32 GetMatchLen(Int32 index, UInt32 distance, UInt32 limit); + UInt32 GetNumAvailableBytes(); + } + + interface IMatchFinder : IInWindowStream + { + void Create(UInt32 historySize, UInt32 keepAddBufferBefore, + UInt32 matchMaxLen, UInt32 keepAddBufferAfter); + UInt32 GetMatches(UInt32[] distances); + void Skip(UInt32 num); + } +} diff --git a/src/Microsoft.DotNet.Archive/LZMA/Compress/LZ/LzBinTree.cs b/src/Microsoft.DotNet.Archive/LZMA/Compress/LZ/LzBinTree.cs new file mode 100644 index 0000000000..017cf8ea2f --- /dev/null +++ b/src/Microsoft.DotNet.Archive/LZMA/Compress/LZ/LzBinTree.cs @@ -0,0 +1,370 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +// LzBinTree.cs + +using System; + +namespace SevenZip.Compression.LZ +{ + public class BinTree : InWindow, IMatchFinder + { + UInt32 _cyclicBufferPos; + UInt32 _cyclicBufferSize = 0; + UInt32 _matchMaxLen; + + UInt32[] _son; + UInt32[] _hash; + + UInt32 _cutValue = 0xFF; + UInt32 _hashMask; + UInt32 _hashSizeSum = 0; + + bool HASH_ARRAY = true; + + const UInt32 kHash2Size = 1 << 10; + const UInt32 kHash3Size = 1 << 16; + const UInt32 kBT2HashSize = 1 << 16; + const UInt32 kStartMaxLen = 1; + const UInt32 kHash3Offset = kHash2Size; + const UInt32 kEmptyHashValue = 0; + const UInt32 kMaxValForNormalize = ((UInt32)1 << 31) - 1; + + UInt32 kNumHashDirectBytes = 0; + UInt32 kMinMatchCheck = 4; + UInt32 kFixHashSize = kHash2Size + kHash3Size; + + public void SetType(int numHashBytes) + { + HASH_ARRAY = (numHashBytes > 2); + if (HASH_ARRAY) + { + kNumHashDirectBytes = 0; + kMinMatchCheck = 4; + kFixHashSize = kHash2Size + kHash3Size; + } + else + { + kNumHashDirectBytes = 2; + kMinMatchCheck = 2 + 1; + kFixHashSize = 0; + } + } + + public new void SetStream(System.IO.Stream stream) { base.SetStream(stream); } + public new void ReleaseStream() { base.ReleaseStream(); } + + public new void Init() + { + base.Init(); + for (UInt32 i = 0; i < _hashSizeSum; i++) + _hash[i] = kEmptyHashValue; + _cyclicBufferPos = 0; + ReduceOffsets(-1); + } + + public new void MovePos() + { + if (++_cyclicBufferPos >= _cyclicBufferSize) + _cyclicBufferPos = 0; + base.MovePos(); + if (_pos == kMaxValForNormalize) + Normalize(); + } + + public new Byte GetIndexByte(Int32 index) { return base.GetIndexByte(index); } + + public new UInt32 GetMatchLen(Int32 index, UInt32 distance, UInt32 limit) + { return base.GetMatchLen(index, distance, limit); } + + public new UInt32 GetNumAvailableBytes() { return base.GetNumAvailableBytes(); } + + public void Create(UInt32 historySize, UInt32 keepAddBufferBefore, + UInt32 matchMaxLen, UInt32 keepAddBufferAfter) + { + if (historySize > kMaxValForNormalize - 256) + throw new Exception(); + _cutValue = 16 + (matchMaxLen >> 1); + + UInt32 windowReservSize = (historySize + keepAddBufferBefore + + matchMaxLen + keepAddBufferAfter) / 2 + 256; + + base.Create(historySize + keepAddBufferBefore, matchMaxLen + keepAddBufferAfter, windowReservSize); + + _matchMaxLen = matchMaxLen; + + UInt32 cyclicBufferSize = historySize + 1; + if (_cyclicBufferSize != cyclicBufferSize) + _son = new UInt32[(_cyclicBufferSize = cyclicBufferSize) * 2]; + + UInt32 hs = kBT2HashSize; + + if (HASH_ARRAY) + { + hs = historySize - 1; + hs |= (hs >> 1); + hs |= (hs >> 2); + hs |= (hs >> 4); + hs |= (hs >> 8); + hs >>= 1; + hs |= 0xFFFF; + if (hs > (1 << 24)) + hs >>= 1; + _hashMask = hs; + hs++; + hs += kFixHashSize; + } + if (hs != _hashSizeSum) + _hash = new UInt32[_hashSizeSum = hs]; + } + + public UInt32 GetMatches(UInt32[] distances) + { + UInt32 lenLimit; + if (_pos + _matchMaxLen <= _streamPos) + lenLimit = _matchMaxLen; + else + { + lenLimit = _streamPos - _pos; + if (lenLimit < kMinMatchCheck) + { + MovePos(); + return 0; + } + } + + UInt32 offset = 0; + UInt32 matchMinPos = (_pos > _cyclicBufferSize) ? (_pos - _cyclicBufferSize) : 0; + UInt32 cur = _bufferOffset + _pos; + UInt32 maxLen = kStartMaxLen; // to avoid items for len < hashSize; + UInt32 hashValue, hash2Value = 0, hash3Value = 0; + + if (HASH_ARRAY) + { + UInt32 temp = CRC.Table[_bufferBase[cur]] ^ _bufferBase[cur + 1]; + hash2Value = temp & (kHash2Size - 1); + temp ^= ((UInt32)(_bufferBase[cur + 2]) << 8); + hash3Value = temp & (kHash3Size - 1); + hashValue = (temp ^ (CRC.Table[_bufferBase[cur + 3]] << 5)) & _hashMask; + } + else + hashValue = _bufferBase[cur] ^ ((UInt32)(_bufferBase[cur + 1]) << 8); + + UInt32 curMatch = _hash[kFixHashSize + hashValue]; + if (HASH_ARRAY) + { + UInt32 curMatch2 = _hash[hash2Value]; + UInt32 curMatch3 = _hash[kHash3Offset + hash3Value]; + _hash[hash2Value] = _pos; + _hash[kHash3Offset + hash3Value] = _pos; + if (curMatch2 > matchMinPos) + if (_bufferBase[_bufferOffset + curMatch2] == _bufferBase[cur]) + { + distances[offset++] = maxLen = 2; + distances[offset++] = _pos - curMatch2 - 1; + } + if (curMatch3 > matchMinPos) + if (_bufferBase[_bufferOffset + curMatch3] == _bufferBase[cur]) + { + if (curMatch3 == curMatch2) + offset -= 2; + distances[offset++] = maxLen = 3; + distances[offset++] = _pos - curMatch3 - 1; + curMatch2 = curMatch3; + } + if (offset != 0 && curMatch2 == curMatch) + { + offset -= 2; + maxLen = kStartMaxLen; + } + } + + _hash[kFixHashSize + hashValue] = _pos; + + UInt32 ptr0 = (_cyclicBufferPos << 1) + 1; + UInt32 ptr1 = (_cyclicBufferPos << 1); + + UInt32 len0, len1; + len0 = len1 = kNumHashDirectBytes; + + if (kNumHashDirectBytes != 0) + { + if (curMatch > matchMinPos) + { + if (_bufferBase[_bufferOffset + curMatch + kNumHashDirectBytes] != + _bufferBase[cur + kNumHashDirectBytes]) + { + distances[offset++] = maxLen = kNumHashDirectBytes; + distances[offset++] = _pos - curMatch - 1; + } + } + } + + UInt32 count = _cutValue; + + while(true) + { + if(curMatch <= matchMinPos || count-- == 0) + { + _son[ptr0] = _son[ptr1] = kEmptyHashValue; + break; + } + UInt32 delta = _pos - curMatch; + UInt32 cyclicPos = ((delta <= _cyclicBufferPos) ? + (_cyclicBufferPos - delta) : + (_cyclicBufferPos - delta + _cyclicBufferSize)) << 1; + + UInt32 pby1 = _bufferOffset + curMatch; + UInt32 len = Math.Min(len0, len1); + if (_bufferBase[pby1 + len] == _bufferBase[cur + len]) + { + while(++len != lenLimit) + if (_bufferBase[pby1 + len] != _bufferBase[cur + len]) + break; + if (maxLen < len) + { + distances[offset++] = maxLen = len; + distances[offset++] = delta - 1; + if (len == lenLimit) + { + _son[ptr1] = _son[cyclicPos]; + _son[ptr0] = _son[cyclicPos + 1]; + break; + } + } + } + if (_bufferBase[pby1 + len] < _bufferBase[cur + len]) + { + _son[ptr1] = curMatch; + ptr1 = cyclicPos + 1; + curMatch = _son[ptr1]; + len1 = len; + } + else + { + _son[ptr0] = curMatch; + ptr0 = cyclicPos; + curMatch = _son[ptr0]; + len0 = len; + } + } + MovePos(); + return offset; + } + + public void Skip(UInt32 num) + { + do + { + UInt32 lenLimit; + if (_pos + _matchMaxLen <= _streamPos) + lenLimit = _matchMaxLen; + else + { + lenLimit = _streamPos - _pos; + if (lenLimit < kMinMatchCheck) + { + MovePos(); + continue; + } + } + + UInt32 matchMinPos = (_pos > _cyclicBufferSize) ? (_pos - _cyclicBufferSize) : 0; + UInt32 cur = _bufferOffset + _pos; + + UInt32 hashValue; + + if (HASH_ARRAY) + { + UInt32 temp = CRC.Table[_bufferBase[cur]] ^ _bufferBase[cur + 1]; + UInt32 hash2Value = temp & (kHash2Size - 1); + _hash[hash2Value] = _pos; + temp ^= ((UInt32)(_bufferBase[cur + 2]) << 8); + UInt32 hash3Value = temp & (kHash3Size - 1); + _hash[kHash3Offset + hash3Value] = _pos; + hashValue = (temp ^ (CRC.Table[_bufferBase[cur + 3]] << 5)) & _hashMask; + } + else + hashValue = _bufferBase[cur] ^ ((UInt32)(_bufferBase[cur + 1]) << 8); + + UInt32 curMatch = _hash[kFixHashSize + hashValue]; + _hash[kFixHashSize + hashValue] = _pos; + + UInt32 ptr0 = (_cyclicBufferPos << 1) + 1; + UInt32 ptr1 = (_cyclicBufferPos << 1); + + UInt32 len0, len1; + len0 = len1 = kNumHashDirectBytes; + + UInt32 count = _cutValue; + while (true) + { + if (curMatch <= matchMinPos || count-- == 0) + { + _son[ptr0] = _son[ptr1] = kEmptyHashValue; + break; + } + + UInt32 delta = _pos - curMatch; + UInt32 cyclicPos = ((delta <= _cyclicBufferPos) ? + (_cyclicBufferPos - delta) : + (_cyclicBufferPos - delta + _cyclicBufferSize)) << 1; + + UInt32 pby1 = _bufferOffset + curMatch; + UInt32 len = Math.Min(len0, len1); + if (_bufferBase[pby1 + len] == _bufferBase[cur + len]) + { + while (++len != lenLimit) + if (_bufferBase[pby1 + len] != _bufferBase[cur + len]) + break; + if (len == lenLimit) + { + _son[ptr1] = _son[cyclicPos]; + _son[ptr0] = _son[cyclicPos + 1]; + break; + } + } + if (_bufferBase[pby1 + len] < _bufferBase[cur + len]) + { + _son[ptr1] = curMatch; + ptr1 = cyclicPos + 1; + curMatch = _son[ptr1]; + len1 = len; + } + else + { + _son[ptr0] = curMatch; + ptr0 = cyclicPos; + curMatch = _son[ptr0]; + len0 = len; + } + } + MovePos(); + } + while (--num != 0); + } + + void NormalizeLinks(UInt32[] items, UInt32 numItems, UInt32 subValue) + { + for (UInt32 i = 0; i < numItems; i++) + { + UInt32 value = items[i]; + if (value <= subValue) + value = kEmptyHashValue; + else + value -= subValue; + items[i] = value; + } + } + + void Normalize() + { + UInt32 subValue = _pos - _cyclicBufferSize; + NormalizeLinks(_son, _cyclicBufferSize * 2, subValue); + NormalizeLinks(_hash, _hashSizeSum, subValue); + ReduceOffsets((Int32)subValue); + } + + public void SetCutValue(UInt32 cutValue) { _cutValue = cutValue; } + } +} diff --git a/src/Microsoft.DotNet.Archive/LZMA/Compress/LZ/LzInWindow.cs b/src/Microsoft.DotNet.Archive/LZMA/Compress/LZ/LzInWindow.cs new file mode 100644 index 0000000000..1ee8282f11 --- /dev/null +++ b/src/Microsoft.DotNet.Archive/LZMA/Compress/LZ/LzInWindow.cs @@ -0,0 +1,135 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +// LzInWindow.cs + +using System; + +namespace SevenZip.Compression.LZ +{ + public class InWindow + { + public Byte[] _bufferBase = null; // pointer to buffer with data + System.IO.Stream _stream; + UInt32 _posLimit; // offset (from _buffer) of first byte when new block reading must be done + bool _streamEndWasReached; // if (true) then _streamPos shows real end of stream + + UInt32 _pointerToLastSafePosition; + + public UInt32 _bufferOffset; + + public UInt32 _blockSize; // Size of Allocated memory block + public UInt32 _pos; // offset (from _buffer) of curent byte + UInt32 _keepSizeBefore; // how many BYTEs must be kept in buffer before _pos + UInt32 _keepSizeAfter; // how many BYTEs must be kept buffer after _pos + public UInt32 _streamPos; // offset (from _buffer) of first not read byte from Stream + + public void MoveBlock() + { + UInt32 offset = (UInt32)(_bufferOffset) + _pos - _keepSizeBefore; + // we need one additional byte, since MovePos moves on 1 byte. + if (offset > 0) + offset--; + + UInt32 numBytes = (UInt32)(_bufferOffset) + _streamPos - offset; + + // check negative offset ???? + for (UInt32 i = 0; i < numBytes; i++) + _bufferBase[i] = _bufferBase[offset + i]; + _bufferOffset -= offset; + } + + public virtual void ReadBlock() + { + if (_streamEndWasReached) + return; + while (true) + { + int size = (int)((0 - _bufferOffset) + _blockSize - _streamPos); + if (size == 0) + return; + int numReadBytes = _stream.Read(_bufferBase, (int)(_bufferOffset + _streamPos), size); + if (numReadBytes == 0) + { + _posLimit = _streamPos; + UInt32 pointerToPostion = _bufferOffset + _posLimit; + if (pointerToPostion > _pointerToLastSafePosition) + _posLimit = (UInt32)(_pointerToLastSafePosition - _bufferOffset); + + _streamEndWasReached = true; + return; + } + _streamPos += (UInt32)numReadBytes; + if (_streamPos >= _pos + _keepSizeAfter) + _posLimit = _streamPos - _keepSizeAfter; + } + } + + void Free() { _bufferBase = null; } + + public void Create(UInt32 keepSizeBefore, UInt32 keepSizeAfter, UInt32 keepSizeReserv) + { + _keepSizeBefore = keepSizeBefore; + _keepSizeAfter = keepSizeAfter; + UInt32 blockSize = keepSizeBefore + keepSizeAfter + keepSizeReserv; + if (_bufferBase == null || _blockSize != blockSize) + { + Free(); + _blockSize = blockSize; + _bufferBase = new Byte[_blockSize]; + } + _pointerToLastSafePosition = _blockSize - keepSizeAfter; + } + + public void SetStream(System.IO.Stream stream) { _stream = stream; } + public void ReleaseStream() { _stream = null; } + + public void Init() + { + _bufferOffset = 0; + _pos = 0; + _streamPos = 0; + _streamEndWasReached = false; + ReadBlock(); + } + + public void MovePos() + { + _pos++; + if (_pos > _posLimit) + { + UInt32 pointerToPostion = _bufferOffset + _pos; + if (pointerToPostion > _pointerToLastSafePosition) + MoveBlock(); + ReadBlock(); + } + } + + public Byte GetIndexByte(Int32 index) { return _bufferBase[_bufferOffset + _pos + index]; } + + // index + limit have not to exceed _keepSizeAfter; + public UInt32 GetMatchLen(Int32 index, UInt32 distance, UInt32 limit) + { + if (_streamEndWasReached) + if ((_pos + index) + limit > _streamPos) + limit = _streamPos - (UInt32)(_pos + index); + distance++; + // Byte *pby = _buffer + (size_t)_pos + index; + UInt32 pby = _bufferOffset + _pos + (UInt32)index; + + UInt32 i; + for (i = 0; i < limit && _bufferBase[pby + i] == _bufferBase[pby + i - distance]; i++); + return i; + } + + public UInt32 GetNumAvailableBytes() { return _streamPos - _pos; } + + public void ReduceOffsets(Int32 subValue) + { + _bufferOffset += (UInt32)subValue; + _posLimit -= (UInt32)subValue; + _pos -= (UInt32)subValue; + _streamPos -= (UInt32)subValue; + } + } +} diff --git a/src/Microsoft.DotNet.Archive/LZMA/Compress/LZ/LzOutWindow.cs b/src/Microsoft.DotNet.Archive/LZMA/Compress/LZ/LzOutWindow.cs new file mode 100644 index 0000000000..479ae4f130 --- /dev/null +++ b/src/Microsoft.DotNet.Archive/LZMA/Compress/LZ/LzOutWindow.cs @@ -0,0 +1,113 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +// LzOutWindow.cs + +namespace SevenZip.Compression.LZ +{ + public class OutWindow + { + byte[] _buffer = null; + uint _pos; + uint _windowSize = 0; + uint _streamPos; + System.IO.Stream _stream; + + public uint TrainSize = 0; + + public void Create(uint windowSize) + { + if (_windowSize != windowSize) + { + // System.GC.Collect(); + _buffer = new byte[windowSize]; + } + _windowSize = windowSize; + _pos = 0; + _streamPos = 0; + } + + public void Init(System.IO.Stream stream, bool solid) + { + ReleaseStream(); + _stream = stream; + if (!solid) + { + _streamPos = 0; + _pos = 0; + TrainSize = 0; + } + } + + public bool Train(System.IO.Stream stream) + { + long len = stream.Length; + uint size = (len < _windowSize) ? (uint)len : _windowSize; + TrainSize = size; + stream.Position = len - size; + _streamPos = _pos = 0; + while (size > 0) + { + uint curSize = _windowSize - _pos; + if (size < curSize) + curSize = size; + int numReadBytes = stream.Read(_buffer, (int)_pos, (int)curSize); + if (numReadBytes == 0) + return false; + size -= (uint)numReadBytes; + _pos += (uint)numReadBytes; + _streamPos += (uint)numReadBytes; + if (_pos == _windowSize) + _streamPos = _pos = 0; + } + return true; + } + + public void ReleaseStream() + { + Flush(); + _stream = null; + } + + public void Flush() + { + uint size = _pos - _streamPos; + if (size == 0) + return; + _stream.Write(_buffer, (int)_streamPos, (int)size); + if (_pos >= _windowSize) + _pos = 0; + _streamPos = _pos; + } + + public void CopyBlock(uint distance, uint len) + { + uint pos = _pos - distance - 1; + if (pos >= _windowSize) + pos += _windowSize; + for (; len > 0; len--) + { + if (pos >= _windowSize) + pos = 0; + _buffer[_pos++] = _buffer[pos++]; + if (_pos >= _windowSize) + Flush(); + } + } + + public void PutByte(byte b) + { + _buffer[_pos++] = b; + if (_pos >= _windowSize) + Flush(); + } + + public byte GetByte(uint distance) + { + uint pos = _pos - distance - 1; + if (pos >= _windowSize) + pos += _windowSize; + return _buffer[pos]; + } + } +} diff --git a/src/Microsoft.DotNet.Archive/LZMA/Compress/LZMA/LzmaBase.cs b/src/Microsoft.DotNet.Archive/LZMA/Compress/LZMA/LzmaBase.cs new file mode 100644 index 0000000000..f4a8f823f0 --- /dev/null +++ b/src/Microsoft.DotNet.Archive/LZMA/Compress/LZMA/LzmaBase.cs @@ -0,0 +1,79 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +// LzmaBase.cs + +namespace SevenZip.Compression.LZMA +{ + internal abstract class Base + { + public const uint kNumRepDistances = 4; + public const uint kNumStates = 12; + + // static byte []kLiteralNextStates = {0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 4, 5}; + // static byte []kMatchNextStates = {7, 7, 7, 7, 7, 7, 7, 10, 10, 10, 10, 10}; + // static byte []kRepNextStates = {8, 8, 8, 8, 8, 8, 8, 11, 11, 11, 11, 11}; + // static byte []kShortRepNextStates = {9, 9, 9, 9, 9, 9, 9, 11, 11, 11, 11, 11}; + + public struct State + { + public uint Index; + public void Init() { Index = 0; } + public void UpdateChar() + { + if (Index < 4) Index = 0; + else if (Index < 10) Index -= 3; + else Index -= 6; + } + public void UpdateMatch() { Index = (uint)(Index < 7 ? 7 : 10); } + public void UpdateRep() { Index = (uint)(Index < 7 ? 8 : 11); } + public void UpdateShortRep() { Index = (uint)(Index < 7 ? 9 : 11); } + public bool IsCharState() { return Index < 7; } + } + + public const int kNumPosSlotBits = 6; + public const int kDicLogSizeMin = 0; + // public const int kDicLogSizeMax = 30; + // public const uint kDistTableSizeMax = kDicLogSizeMax * 2; + + public const int kNumLenToPosStatesBits = 2; // it's for speed optimization + public const uint kNumLenToPosStates = 1 << kNumLenToPosStatesBits; + + public const uint kMatchMinLen = 2; + + public static uint GetLenToPosState(uint len) + { + len -= kMatchMinLen; + if (len < kNumLenToPosStates) + return len; + return (uint)(kNumLenToPosStates - 1); + } + + public const int kNumAlignBits = 4; + public const uint kAlignTableSize = 1 << kNumAlignBits; + public const uint kAlignMask = (kAlignTableSize - 1); + + public const uint kStartPosModelIndex = 4; + public const uint kEndPosModelIndex = 14; + public const uint kNumPosModels = kEndPosModelIndex - kStartPosModelIndex; + + public const uint kNumFullDistances = 1 << ((int)kEndPosModelIndex / 2); + + public const uint kNumLitPosStatesBitsEncodingMax = 4; + public const uint kNumLitContextBitsMax = 8; + + public const int kNumPosStatesBitsMax = 4; + public const uint kNumPosStatesMax = (1 << kNumPosStatesBitsMax); + public const int kNumPosStatesBitsEncodingMax = 4; + public const uint kNumPosStatesEncodingMax = (1 << kNumPosStatesBitsEncodingMax); + + public const int kNumLowLenBits = 3; + public const int kNumMidLenBits = 3; + public const int kNumHighLenBits = 8; + public const uint kNumLowLenSymbols = 1 << kNumLowLenBits; + public const uint kNumMidLenSymbols = 1 << kNumMidLenBits; + public const uint kNumLenSymbols = kNumLowLenSymbols + kNumMidLenSymbols + + (1 << kNumHighLenBits); + public const uint kMatchMaxLen = kMatchMinLen + kNumLenSymbols - 1; + } +} diff --git a/src/Microsoft.DotNet.Archive/LZMA/Compress/LZMA/LzmaDecoder.cs b/src/Microsoft.DotNet.Archive/LZMA/Compress/LZMA/LzmaDecoder.cs new file mode 100644 index 0000000000..95d42eed06 --- /dev/null +++ b/src/Microsoft.DotNet.Archive/LZMA/Compress/LZMA/LzmaDecoder.cs @@ -0,0 +1,402 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +// LzmaDecoder.cs + +using System; + +namespace SevenZip.Compression.LZMA +{ + using RangeCoder; + + public class Decoder : ICoder, ISetDecoderProperties // ,System.IO.Stream + { + class LenDecoder + { + BitDecoder m_Choice = new BitDecoder(); + BitDecoder m_Choice2 = new BitDecoder(); + BitTreeDecoder[] m_LowCoder = new BitTreeDecoder[Base.kNumPosStatesMax]; + BitTreeDecoder[] m_MidCoder = new BitTreeDecoder[Base.kNumPosStatesMax]; + BitTreeDecoder m_HighCoder = new BitTreeDecoder(Base.kNumHighLenBits); + uint m_NumPosStates = 0; + + public void Create(uint numPosStates) + { + for (uint posState = m_NumPosStates; posState < numPosStates; posState++) + { + m_LowCoder[posState] = new BitTreeDecoder(Base.kNumLowLenBits); + m_MidCoder[posState] = new BitTreeDecoder(Base.kNumMidLenBits); + } + m_NumPosStates = numPosStates; + } + + public void Init() + { + m_Choice.Init(); + for (uint posState = 0; posState < m_NumPosStates; posState++) + { + m_LowCoder[posState].Init(); + m_MidCoder[posState].Init(); + } + m_Choice2.Init(); + m_HighCoder.Init(); + } + + public uint Decode(RangeCoder.Decoder rangeDecoder, uint posState) + { + if (m_Choice.Decode(rangeDecoder) == 0) + return m_LowCoder[posState].Decode(rangeDecoder); + else + { + uint symbol = Base.kNumLowLenSymbols; + if (m_Choice2.Decode(rangeDecoder) == 0) + symbol += m_MidCoder[posState].Decode(rangeDecoder); + else + { + symbol += Base.kNumMidLenSymbols; + symbol += m_HighCoder.Decode(rangeDecoder); + } + return symbol; + } + } + } + + class LiteralDecoder + { + struct Decoder2 + { + BitDecoder[] m_Decoders; + public void Create() { m_Decoders = new BitDecoder[0x300]; } + public void Init() { for (int i = 0; i < 0x300; i++) m_Decoders[i].Init(); } + + public byte DecodeNormal(RangeCoder.Decoder rangeDecoder) + { + uint symbol = 1; + do + symbol = (symbol << 1) | m_Decoders[symbol].Decode(rangeDecoder); + while (symbol < 0x100); + return (byte)symbol; + } + + public byte DecodeWithMatchByte(RangeCoder.Decoder rangeDecoder, byte matchByte) + { + uint symbol = 1; + do + { + uint matchBit = (uint)(matchByte >> 7) & 1; + matchByte <<= 1; + uint bit = m_Decoders[((1 + matchBit) << 8) + symbol].Decode(rangeDecoder); + symbol = (symbol << 1) | bit; + if (matchBit != bit) + { + while (symbol < 0x100) + symbol = (symbol << 1) | m_Decoders[symbol].Decode(rangeDecoder); + break; + } + } + while (symbol < 0x100); + return (byte)symbol; + } + } + + Decoder2[] m_Coders; + int m_NumPrevBits; + int m_NumPosBits; + uint m_PosMask; + + public void Create(int numPosBits, int numPrevBits) + { + if (m_Coders != null && m_NumPrevBits == numPrevBits && + m_NumPosBits == numPosBits) + return; + m_NumPosBits = numPosBits; + m_PosMask = ((uint)1 << numPosBits) - 1; + m_NumPrevBits = numPrevBits; + uint numStates = (uint)1 << (m_NumPrevBits + m_NumPosBits); + m_Coders = new Decoder2[numStates]; + for (uint i = 0; i < numStates; i++) + m_Coders[i].Create(); + } + + public void Init() + { + uint numStates = (uint)1 << (m_NumPrevBits + m_NumPosBits); + for (uint i = 0; i < numStates; i++) + m_Coders[i].Init(); + } + + uint GetState(uint pos, byte prevByte) + { return ((pos & m_PosMask) << m_NumPrevBits) + (uint)(prevByte >> (8 - m_NumPrevBits)); } + + public byte DecodeNormal(RangeCoder.Decoder rangeDecoder, uint pos, byte prevByte) + { return m_Coders[GetState(pos, prevByte)].DecodeNormal(rangeDecoder); } + + public byte DecodeWithMatchByte(RangeCoder.Decoder rangeDecoder, uint pos, byte prevByte, byte matchByte) + { return m_Coders[GetState(pos, prevByte)].DecodeWithMatchByte(rangeDecoder, matchByte); } + }; + + LZ.OutWindow m_OutWindow = new LZ.OutWindow(); + RangeCoder.Decoder m_RangeDecoder = new RangeCoder.Decoder(); + + BitDecoder[] m_IsMatchDecoders = new BitDecoder[Base.kNumStates << Base.kNumPosStatesBitsMax]; + BitDecoder[] m_IsRepDecoders = new BitDecoder[Base.kNumStates]; + BitDecoder[] m_IsRepG0Decoders = new BitDecoder[Base.kNumStates]; + BitDecoder[] m_IsRepG1Decoders = new BitDecoder[Base.kNumStates]; + BitDecoder[] m_IsRepG2Decoders = new BitDecoder[Base.kNumStates]; + BitDecoder[] m_IsRep0LongDecoders = new BitDecoder[Base.kNumStates << Base.kNumPosStatesBitsMax]; + + BitTreeDecoder[] m_PosSlotDecoder = new BitTreeDecoder[Base.kNumLenToPosStates]; + BitDecoder[] m_PosDecoders = new BitDecoder[Base.kNumFullDistances - Base.kEndPosModelIndex]; + + BitTreeDecoder m_PosAlignDecoder = new BitTreeDecoder(Base.kNumAlignBits); + + LenDecoder m_LenDecoder = new LenDecoder(); + LenDecoder m_RepLenDecoder = new LenDecoder(); + + LiteralDecoder m_LiteralDecoder = new LiteralDecoder(); + + uint m_DictionarySize; + uint m_DictionarySizeCheck; + + uint m_PosStateMask; + + public Decoder() + { + m_DictionarySize = 0xFFFFFFFF; + for (int i = 0; i < Base.kNumLenToPosStates; i++) + m_PosSlotDecoder[i] = new BitTreeDecoder(Base.kNumPosSlotBits); + } + + void SetDictionarySize(uint dictionarySize) + { + if (m_DictionarySize != dictionarySize) + { + m_DictionarySize = dictionarySize; + m_DictionarySizeCheck = Math.Max(m_DictionarySize, 1); + uint blockSize = Math.Max(m_DictionarySizeCheck, (1 << 12)); + m_OutWindow.Create(blockSize); + } + } + + void SetLiteralProperties(int lp, int lc) + { + if (lp > 8) + throw new InvalidParamException(); + if (lc > 8) + throw new InvalidParamException(); + m_LiteralDecoder.Create(lp, lc); + } + + void SetPosBitsProperties(int pb) + { + if (pb > Base.kNumPosStatesBitsMax) + throw new InvalidParamException(); + uint numPosStates = (uint)1 << pb; + m_LenDecoder.Create(numPosStates); + m_RepLenDecoder.Create(numPosStates); + m_PosStateMask = numPosStates - 1; + } + + bool _solid = false; + void Init(System.IO.Stream inStream, System.IO.Stream outStream) + { + m_RangeDecoder.Init(inStream); + m_OutWindow.Init(outStream, _solid); + + uint i; + for (i = 0; i < Base.kNumStates; i++) + { + for (uint j = 0; j <= m_PosStateMask; j++) + { + uint index = (i << Base.kNumPosStatesBitsMax) + j; + m_IsMatchDecoders[index].Init(); + m_IsRep0LongDecoders[index].Init(); + } + m_IsRepDecoders[i].Init(); + m_IsRepG0Decoders[i].Init(); + m_IsRepG1Decoders[i].Init(); + m_IsRepG2Decoders[i].Init(); + } + + m_LiteralDecoder.Init(); + for (i = 0; i < Base.kNumLenToPosStates; i++) + m_PosSlotDecoder[i].Init(); + // m_PosSpecDecoder.Init(); + for (i = 0; i < Base.kNumFullDistances - Base.kEndPosModelIndex; i++) + m_PosDecoders[i].Init(); + + m_LenDecoder.Init(); + m_RepLenDecoder.Init(); + m_PosAlignDecoder.Init(); + } + + public void Code(System.IO.Stream inStream, System.IO.Stream outStream, + Int64 inSize, Int64 outSize, ICodeProgress progress) + { + Init(inStream, outStream); + + Base.State state = new Base.State(); + state.Init(); + uint rep0 = 0, rep1 = 0, rep2 = 0, rep3 = 0; + + UInt64 nowPos64 = 0; + UInt64 outSize64 = (UInt64)outSize; + if (nowPos64 < outSize64) + { + if (m_IsMatchDecoders[state.Index << Base.kNumPosStatesBitsMax].Decode(m_RangeDecoder) != 0) + throw new DataErrorException(); + state.UpdateChar(); + byte b = m_LiteralDecoder.DecodeNormal(m_RangeDecoder, 0, 0); + m_OutWindow.PutByte(b); + nowPos64++; + } + while (nowPos64 < outSize64) + { + progress.SetProgress(inStream.Position, (long)nowPos64); + // UInt64 next = Math.Min(nowPos64 + (1 << 18), outSize64); + // while(nowPos64 < next) + { + uint posState = (uint)nowPos64 & m_PosStateMask; + if (m_IsMatchDecoders[(state.Index << Base.kNumPosStatesBitsMax) + posState].Decode(m_RangeDecoder) == 0) + { + byte b; + byte prevByte = m_OutWindow.GetByte(0); + if (!state.IsCharState()) + b = m_LiteralDecoder.DecodeWithMatchByte(m_RangeDecoder, + (uint)nowPos64, prevByte, m_OutWindow.GetByte(rep0)); + else + b = m_LiteralDecoder.DecodeNormal(m_RangeDecoder, (uint)nowPos64, prevByte); + m_OutWindow.PutByte(b); + state.UpdateChar(); + nowPos64++; + } + else + { + uint len; + if (m_IsRepDecoders[state.Index].Decode(m_RangeDecoder) == 1) + { + if (m_IsRepG0Decoders[state.Index].Decode(m_RangeDecoder) == 0) + { + if (m_IsRep0LongDecoders[(state.Index << Base.kNumPosStatesBitsMax) + posState].Decode(m_RangeDecoder) == 0) + { + state.UpdateShortRep(); + m_OutWindow.PutByte(m_OutWindow.GetByte(rep0)); + nowPos64++; + continue; + } + } + else + { + UInt32 distance; + if (m_IsRepG1Decoders[state.Index].Decode(m_RangeDecoder) == 0) + { + distance = rep1; + } + else + { + if (m_IsRepG2Decoders[state.Index].Decode(m_RangeDecoder) == 0) + distance = rep2; + else + { + distance = rep3; + rep3 = rep2; + } + rep2 = rep1; + } + rep1 = rep0; + rep0 = distance; + } + len = m_RepLenDecoder.Decode(m_RangeDecoder, posState) + Base.kMatchMinLen; + state.UpdateRep(); + } + else + { + rep3 = rep2; + rep2 = rep1; + rep1 = rep0; + len = Base.kMatchMinLen + m_LenDecoder.Decode(m_RangeDecoder, posState); + state.UpdateMatch(); + uint posSlot = m_PosSlotDecoder[Base.GetLenToPosState(len)].Decode(m_RangeDecoder); + if (posSlot >= Base.kStartPosModelIndex) + { + int numDirectBits = (int)((posSlot >> 1) - 1); + rep0 = ((2 | (posSlot & 1)) << numDirectBits); + if (posSlot < Base.kEndPosModelIndex) + rep0 += BitTreeDecoder.ReverseDecode(m_PosDecoders, + rep0 - posSlot - 1, m_RangeDecoder, numDirectBits); + else + { + rep0 += (m_RangeDecoder.DecodeDirectBits( + numDirectBits - Base.kNumAlignBits) << Base.kNumAlignBits); + rep0 += m_PosAlignDecoder.ReverseDecode(m_RangeDecoder); + } + } + else + rep0 = posSlot; + } + if (rep0 >= m_OutWindow.TrainSize + nowPos64 || rep0 >= m_DictionarySizeCheck) + { + if (rep0 == 0xFFFFFFFF) + break; + throw new DataErrorException(); + } + m_OutWindow.CopyBlock(rep0, len); + nowPos64 += len; + } + } + } + m_OutWindow.Flush(); + m_OutWindow.ReleaseStream(); + m_RangeDecoder.ReleaseStream(); + } + + public void SetDecoderProperties(byte[] properties) + { + if (properties.Length < 5) + throw new InvalidParamException(); + int lc = properties[0] % 9; + int remainder = properties[0] / 9; + int lp = remainder % 5; + int pb = remainder / 5; + if (pb > Base.kNumPosStatesBitsMax) + throw new InvalidParamException(); + UInt32 dictionarySize = 0; + for (int i = 0; i < 4; i++) + dictionarySize += ((UInt32)(properties[1 + i])) << (i * 8); + SetDictionarySize(dictionarySize); + SetLiteralProperties(lp, lc); + SetPosBitsProperties(pb); + } + + public bool Train(System.IO.Stream stream) + { + _solid = true; + return m_OutWindow.Train(stream); + } + + /* + public override bool CanRead { get { return true; }} + public override bool CanWrite { get { return true; }} + public override bool CanSeek { get { return true; }} + public override long Length { get { return 0; }} + public override long Position + { + get { return 0; } + set { } + } + public override void Flush() { } + public override int Read(byte[] buffer, int offset, int count) + { + return 0; + } + public override void Write(byte[] buffer, int offset, int count) + { + } + public override long Seek(long offset, System.IO.SeekOrigin origin) + { + return 0; + } + public override void SetLength(long value) {} + */ + } +} diff --git a/src/Microsoft.DotNet.Archive/LZMA/Compress/LZMA/LzmaEncoder.cs b/src/Microsoft.DotNet.Archive/LZMA/Compress/LZMA/LzmaEncoder.cs new file mode 100644 index 0000000000..527a67e0ca --- /dev/null +++ b/src/Microsoft.DotNet.Archive/LZMA/Compress/LZMA/LzmaEncoder.cs @@ -0,0 +1,1483 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +// LzmaEncoder.cs + +using System; + +namespace SevenZip.Compression.LZMA +{ + using RangeCoder; + + public class Encoder : ICoder, ISetCoderProperties, IWriteCoderProperties + { + enum EMatchFinderType + { + BT2, + BT4, + }; + + const UInt32 kIfinityPrice = 0xFFFFFFF; + + static Byte[] g_FastPos = new Byte[1 << 11]; + + static Encoder() + { + const Byte kFastSlots = 22; + int c = 2; + g_FastPos[0] = 0; + g_FastPos[1] = 1; + for (Byte slotFast = 2; slotFast < kFastSlots; slotFast++) + { + UInt32 k = ((UInt32)1 << ((slotFast >> 1) - 1)); + for (UInt32 j = 0; j < k; j++, c++) + g_FastPos[c] = slotFast; + } + } + + static UInt32 GetPosSlot(UInt32 pos) + { + if (pos < (1 << 11)) + return g_FastPos[pos]; + if (pos < (1 << 21)) + return (UInt32)(g_FastPos[pos >> 10] + 20); + return (UInt32)(g_FastPos[pos >> 20] + 40); + } + + static UInt32 GetPosSlot2(UInt32 pos) + { + if (pos < (1 << 17)) + return (UInt32)(g_FastPos[pos >> 6] + 12); + if (pos < (1 << 27)) + return (UInt32)(g_FastPos[pos >> 16] + 32); + return (UInt32)(g_FastPos[pos >> 26] + 52); + } + + Base.State _state = new Base.State(); + Byte _previousByte; + UInt32[] _repDistances = new UInt32[Base.kNumRepDistances]; + + void BaseInit() + { + _state.Init(); + _previousByte = 0; + for (UInt32 i = 0; i < Base.kNumRepDistances; i++) + _repDistances[i] = 0; + } + + const int kDefaultDictionaryLogSize = 22; + const UInt32 kNumFastBytesDefault = 0x20; + + class LiteralEncoder + { + public struct Encoder2 + { + BitEncoder[] m_Encoders; + + public void Create() { m_Encoders = new BitEncoder[0x300]; } + + public void Init() { for (int i = 0; i < 0x300; i++) m_Encoders[i].Init(); } + + public void Encode(RangeCoder.Encoder rangeEncoder, byte symbol) + { + uint context = 1; + for (int i = 7; i >= 0; i--) + { + uint bit = (uint)((symbol >> i) & 1); + m_Encoders[context].Encode(rangeEncoder, bit); + context = (context << 1) | bit; + } + } + + public void EncodeMatched(RangeCoder.Encoder rangeEncoder, byte matchByte, byte symbol) + { + uint context = 1; + bool same = true; + for (int i = 7; i >= 0; i--) + { + uint bit = (uint)((symbol >> i) & 1); + uint state = context; + if (same) + { + uint matchBit = (uint)((matchByte >> i) & 1); + state += ((1 + matchBit) << 8); + same = (matchBit == bit); + } + m_Encoders[state].Encode(rangeEncoder, bit); + context = (context << 1) | bit; + } + } + + public uint GetPrice(bool matchMode, byte matchByte, byte symbol) + { + uint price = 0; + uint context = 1; + int i = 7; + if (matchMode) + { + for (; i >= 0; i--) + { + uint matchBit = (uint)(matchByte >> i) & 1; + uint bit = (uint)(symbol >> i) & 1; + price += m_Encoders[((1 + matchBit) << 8) + context].GetPrice(bit); + context = (context << 1) | bit; + if (matchBit != bit) + { + i--; + break; + } + } + } + for (; i >= 0; i--) + { + uint bit = (uint)(symbol >> i) & 1; + price += m_Encoders[context].GetPrice(bit); + context = (context << 1) | bit; + } + return price; + } + } + + Encoder2[] m_Coders; + int m_NumPrevBits; + int m_NumPosBits; + uint m_PosMask; + + public void Create(int numPosBits, int numPrevBits) + { + if (m_Coders != null && m_NumPrevBits == numPrevBits && m_NumPosBits == numPosBits) + return; + m_NumPosBits = numPosBits; + m_PosMask = ((uint)1 << numPosBits) - 1; + m_NumPrevBits = numPrevBits; + uint numStates = (uint)1 << (m_NumPrevBits + m_NumPosBits); + m_Coders = new Encoder2[numStates]; + for (uint i = 0; i < numStates; i++) + m_Coders[i].Create(); + } + + public void Init() + { + uint numStates = (uint)1 << (m_NumPrevBits + m_NumPosBits); + for (uint i = 0; i < numStates; i++) + m_Coders[i].Init(); + } + + public Encoder2 GetSubCoder(UInt32 pos, Byte prevByte) + { return m_Coders[((pos & m_PosMask) << m_NumPrevBits) + (uint)(prevByte >> (8 - m_NumPrevBits))]; } + } + + class LenEncoder + { + RangeCoder.BitEncoder _choice = new RangeCoder.BitEncoder(); + RangeCoder.BitEncoder _choice2 = new RangeCoder.BitEncoder(); + RangeCoder.BitTreeEncoder[] _lowCoder = new RangeCoder.BitTreeEncoder[Base.kNumPosStatesEncodingMax]; + RangeCoder.BitTreeEncoder[] _midCoder = new RangeCoder.BitTreeEncoder[Base.kNumPosStatesEncodingMax]; + RangeCoder.BitTreeEncoder _highCoder = new RangeCoder.BitTreeEncoder(Base.kNumHighLenBits); + + public LenEncoder() + { + for (UInt32 posState = 0; posState < Base.kNumPosStatesEncodingMax; posState++) + { + _lowCoder[posState] = new RangeCoder.BitTreeEncoder(Base.kNumLowLenBits); + _midCoder[posState] = new RangeCoder.BitTreeEncoder(Base.kNumMidLenBits); + } + } + + public void Init(UInt32 numPosStates) + { + _choice.Init(); + _choice2.Init(); + for (UInt32 posState = 0; posState < numPosStates; posState++) + { + _lowCoder[posState].Init(); + _midCoder[posState].Init(); + } + _highCoder.Init(); + } + + public void Encode(RangeCoder.Encoder rangeEncoder, UInt32 symbol, UInt32 posState) + { + if (symbol < Base.kNumLowLenSymbols) + { + _choice.Encode(rangeEncoder, 0); + _lowCoder[posState].Encode(rangeEncoder, symbol); + } + else + { + symbol -= Base.kNumLowLenSymbols; + _choice.Encode(rangeEncoder, 1); + if (symbol < Base.kNumMidLenSymbols) + { + _choice2.Encode(rangeEncoder, 0); + _midCoder[posState].Encode(rangeEncoder, symbol); + } + else + { + _choice2.Encode(rangeEncoder, 1); + _highCoder.Encode(rangeEncoder, symbol - Base.kNumMidLenSymbols); + } + } + } + + public void SetPrices(UInt32 posState, UInt32 numSymbols, UInt32[] prices, UInt32 st) + { + UInt32 a0 = _choice.GetPrice0(); + UInt32 a1 = _choice.GetPrice1(); + UInt32 b0 = a1 + _choice2.GetPrice0(); + UInt32 b1 = a1 + _choice2.GetPrice1(); + UInt32 i = 0; + for (i = 0; i < Base.kNumLowLenSymbols; i++) + { + if (i >= numSymbols) + return; + prices[st + i] = a0 + _lowCoder[posState].GetPrice(i); + } + for (; i < Base.kNumLowLenSymbols + Base.kNumMidLenSymbols; i++) + { + if (i >= numSymbols) + return; + prices[st + i] = b0 + _midCoder[posState].GetPrice(i - Base.kNumLowLenSymbols); + } + for (; i < numSymbols; i++) + prices[st + i] = b1 + _highCoder.GetPrice(i - Base.kNumLowLenSymbols - Base.kNumMidLenSymbols); + } + }; + + const UInt32 kNumLenSpecSymbols = Base.kNumLowLenSymbols + Base.kNumMidLenSymbols; + + class LenPriceTableEncoder : LenEncoder + { + UInt32[] _prices = new UInt32[Base.kNumLenSymbols << Base.kNumPosStatesBitsEncodingMax]; + UInt32 _tableSize; + UInt32[] _counters = new UInt32[Base.kNumPosStatesEncodingMax]; + + public void SetTableSize(UInt32 tableSize) { _tableSize = tableSize; } + + public UInt32 GetPrice(UInt32 symbol, UInt32 posState) + { + return _prices[posState * Base.kNumLenSymbols + symbol]; + } + + void UpdateTable(UInt32 posState) + { + SetPrices(posState, _tableSize, _prices, posState * Base.kNumLenSymbols); + _counters[posState] = _tableSize; + } + + public void UpdateTables(UInt32 numPosStates) + { + for (UInt32 posState = 0; posState < numPosStates; posState++) + UpdateTable(posState); + } + + public new void Encode(RangeCoder.Encoder rangeEncoder, UInt32 symbol, UInt32 posState) + { + base.Encode(rangeEncoder, symbol, posState); + if (--_counters[posState] == 0) + UpdateTable(posState); + } + } + + const UInt32 kNumOpts = 1 << 12; + class Optimal + { + public Base.State State; + + public bool Prev1IsChar; + public bool Prev2; + + public UInt32 PosPrev2; + public UInt32 BackPrev2; + + public UInt32 Price; + public UInt32 PosPrev; + public UInt32 BackPrev; + + public UInt32 Backs0; + public UInt32 Backs1; + public UInt32 Backs2; + public UInt32 Backs3; + + public void MakeAsChar() { BackPrev = 0xFFFFFFFF; Prev1IsChar = false; } + public void MakeAsShortRep() { BackPrev = 0; ; Prev1IsChar = false; } + public bool IsShortRep() { return (BackPrev == 0); } + }; + Optimal[] _optimum = new Optimal[kNumOpts]; + LZ.IMatchFinder _matchFinder = null; + RangeCoder.Encoder _rangeEncoder = new RangeCoder.Encoder(); + + RangeCoder.BitEncoder[] _isMatch = new RangeCoder.BitEncoder[Base.kNumStates << Base.kNumPosStatesBitsMax]; + RangeCoder.BitEncoder[] _isRep = new RangeCoder.BitEncoder[Base.kNumStates]; + RangeCoder.BitEncoder[] _isRepG0 = new RangeCoder.BitEncoder[Base.kNumStates]; + RangeCoder.BitEncoder[] _isRepG1 = new RangeCoder.BitEncoder[Base.kNumStates]; + RangeCoder.BitEncoder[] _isRepG2 = new RangeCoder.BitEncoder[Base.kNumStates]; + RangeCoder.BitEncoder[] _isRep0Long = new RangeCoder.BitEncoder[Base.kNumStates << Base.kNumPosStatesBitsMax]; + + RangeCoder.BitTreeEncoder[] _posSlotEncoder = new RangeCoder.BitTreeEncoder[Base.kNumLenToPosStates]; + + RangeCoder.BitEncoder[] _posEncoders = new RangeCoder.BitEncoder[Base.kNumFullDistances - Base.kEndPosModelIndex]; + RangeCoder.BitTreeEncoder _posAlignEncoder = new RangeCoder.BitTreeEncoder(Base.kNumAlignBits); + + LenPriceTableEncoder _lenEncoder = new LenPriceTableEncoder(); + LenPriceTableEncoder _repMatchLenEncoder = new LenPriceTableEncoder(); + + LiteralEncoder _literalEncoder = new LiteralEncoder(); + + UInt32[] _matchDistances = new UInt32[Base.kMatchMaxLen * 2 + 2]; + + UInt32 _numFastBytes = kNumFastBytesDefault; + UInt32 _longestMatchLength; + UInt32 _numDistancePairs; + + UInt32 _additionalOffset; + + UInt32 _optimumEndIndex; + UInt32 _optimumCurrentIndex; + + bool _longestMatchWasFound; + + UInt32[] _posSlotPrices = new UInt32[1 << (Base.kNumPosSlotBits + Base.kNumLenToPosStatesBits)]; + UInt32[] _distancesPrices = new UInt32[Base.kNumFullDistances << Base.kNumLenToPosStatesBits]; + UInt32[] _alignPrices = new UInt32[Base.kAlignTableSize]; + UInt32 _alignPriceCount; + + UInt32 _distTableSize = (kDefaultDictionaryLogSize * 2); + + int _posStateBits = 2; + UInt32 _posStateMask = (4 - 1); + int _numLiteralPosStateBits = 0; + int _numLiteralContextBits = 3; + + UInt32 _dictionarySize = (1 << kDefaultDictionaryLogSize); + UInt32 _dictionarySizePrev = 0xFFFFFFFF; + UInt32 _numFastBytesPrev = 0xFFFFFFFF; + + Int64 nowPos64; + bool _finished; + System.IO.Stream _inStream; + + EMatchFinderType _matchFinderType = EMatchFinderType.BT4; + bool _writeEndMark = false; + + bool _needReleaseMFStream; + + void Create() + { + if (_matchFinder == null) + { + LZ.BinTree bt = new LZ.BinTree(); + int numHashBytes = 4; + if (_matchFinderType == EMatchFinderType.BT2) + numHashBytes = 2; + bt.SetType(numHashBytes); + _matchFinder = bt; + } + _literalEncoder.Create(_numLiteralPosStateBits, _numLiteralContextBits); + + if (_dictionarySize == _dictionarySizePrev && _numFastBytesPrev == _numFastBytes) + return; + _matchFinder.Create(_dictionarySize, kNumOpts, _numFastBytes, Base.kMatchMaxLen + 1); + _dictionarySizePrev = _dictionarySize; + _numFastBytesPrev = _numFastBytes; + } + + public Encoder() + { + for (int i = 0; i < kNumOpts; i++) + _optimum[i] = new Optimal(); + for (int i = 0; i < Base.kNumLenToPosStates; i++) + _posSlotEncoder[i] = new RangeCoder.BitTreeEncoder(Base.kNumPosSlotBits); + } + + void SetWriteEndMarkerMode(bool writeEndMarker) + { + _writeEndMark = writeEndMarker; + } + + void Init() + { + BaseInit(); + _rangeEncoder.Init(); + + uint i; + for (i = 0; i < Base.kNumStates; i++) + { + for (uint j = 0; j <= _posStateMask; j++) + { + uint complexState = (i << Base.kNumPosStatesBitsMax) + j; + _isMatch[complexState].Init(); + _isRep0Long[complexState].Init(); + } + _isRep[i].Init(); + _isRepG0[i].Init(); + _isRepG1[i].Init(); + _isRepG2[i].Init(); + } + _literalEncoder.Init(); + for (i = 0; i < Base.kNumLenToPosStates; i++) + _posSlotEncoder[i].Init(); + for (i = 0; i < Base.kNumFullDistances - Base.kEndPosModelIndex; i++) + _posEncoders[i].Init(); + + _lenEncoder.Init((UInt32)1 << _posStateBits); + _repMatchLenEncoder.Init((UInt32)1 << _posStateBits); + + _posAlignEncoder.Init(); + + _longestMatchWasFound = false; + _optimumEndIndex = 0; + _optimumCurrentIndex = 0; + _additionalOffset = 0; + } + + void ReadMatchDistances(out UInt32 lenRes, out UInt32 numDistancePairs) + { + lenRes = 0; + numDistancePairs = _matchFinder.GetMatches(_matchDistances); + if (numDistancePairs > 0) + { + lenRes = _matchDistances[numDistancePairs - 2]; + if (lenRes == _numFastBytes) + lenRes += _matchFinder.GetMatchLen((int)lenRes - 1, _matchDistances[numDistancePairs - 1], + Base.kMatchMaxLen - lenRes); + } + _additionalOffset++; + } + + + void MovePos(UInt32 num) + { + if (num > 0) + { + _matchFinder.Skip(num); + _additionalOffset += num; + } + } + + UInt32 GetRepLen1Price(Base.State state, UInt32 posState) + { + return _isRepG0[state.Index].GetPrice0() + + _isRep0Long[(state.Index << Base.kNumPosStatesBitsMax) + posState].GetPrice0(); + } + + UInt32 GetPureRepPrice(UInt32 repIndex, Base.State state, UInt32 posState) + { + UInt32 price; + if (repIndex == 0) + { + price = _isRepG0[state.Index].GetPrice0(); + price += _isRep0Long[(state.Index << Base.kNumPosStatesBitsMax) + posState].GetPrice1(); + } + else + { + price = _isRepG0[state.Index].GetPrice1(); + if (repIndex == 1) + price += _isRepG1[state.Index].GetPrice0(); + else + { + price += _isRepG1[state.Index].GetPrice1(); + price += _isRepG2[state.Index].GetPrice(repIndex - 2); + } + } + return price; + } + + UInt32 GetRepPrice(UInt32 repIndex, UInt32 len, Base.State state, UInt32 posState) + { + UInt32 price = _repMatchLenEncoder.GetPrice(len - Base.kMatchMinLen, posState); + return price + GetPureRepPrice(repIndex, state, posState); + } + + UInt32 GetPosLenPrice(UInt32 pos, UInt32 len, UInt32 posState) + { + UInt32 price; + UInt32 lenToPosState = Base.GetLenToPosState(len); + if (pos < Base.kNumFullDistances) + price = _distancesPrices[(lenToPosState * Base.kNumFullDistances) + pos]; + else + price = _posSlotPrices[(lenToPosState << Base.kNumPosSlotBits) + GetPosSlot2(pos)] + + _alignPrices[pos & Base.kAlignMask]; + return price + _lenEncoder.GetPrice(len - Base.kMatchMinLen, posState); + } + + UInt32 Backward(out UInt32 backRes, UInt32 cur) + { + _optimumEndIndex = cur; + UInt32 posMem = _optimum[cur].PosPrev; + UInt32 backMem = _optimum[cur].BackPrev; + do + { + if (_optimum[cur].Prev1IsChar) + { + _optimum[posMem].MakeAsChar(); + _optimum[posMem].PosPrev = posMem - 1; + if (_optimum[cur].Prev2) + { + _optimum[posMem - 1].Prev1IsChar = false; + _optimum[posMem - 1].PosPrev = _optimum[cur].PosPrev2; + _optimum[posMem - 1].BackPrev = _optimum[cur].BackPrev2; + } + } + UInt32 posPrev = posMem; + UInt32 backCur = backMem; + + backMem = _optimum[posPrev].BackPrev; + posMem = _optimum[posPrev].PosPrev; + + _optimum[posPrev].BackPrev = backCur; + _optimum[posPrev].PosPrev = cur; + cur = posPrev; + } + while (cur > 0); + backRes = _optimum[0].BackPrev; + _optimumCurrentIndex = _optimum[0].PosPrev; + return _optimumCurrentIndex; + } + + UInt32[] reps = new UInt32[Base.kNumRepDistances]; + UInt32[] repLens = new UInt32[Base.kNumRepDistances]; + + + UInt32 GetOptimum(UInt32 position, out UInt32 backRes) + { + if (_optimumEndIndex != _optimumCurrentIndex) + { + UInt32 lenRes = _optimum[_optimumCurrentIndex].PosPrev - _optimumCurrentIndex; + backRes = _optimum[_optimumCurrentIndex].BackPrev; + _optimumCurrentIndex = _optimum[_optimumCurrentIndex].PosPrev; + return lenRes; + } + _optimumCurrentIndex = _optimumEndIndex = 0; + + UInt32 lenMain, numDistancePairs; + if (!_longestMatchWasFound) + { + ReadMatchDistances(out lenMain, out numDistancePairs); + } + else + { + lenMain = _longestMatchLength; + numDistancePairs = _numDistancePairs; + _longestMatchWasFound = false; + } + + UInt32 numAvailableBytes = _matchFinder.GetNumAvailableBytes() + 1; + if (numAvailableBytes < 2) + { + backRes = 0xFFFFFFFF; + return 1; + } + if (numAvailableBytes > Base.kMatchMaxLen) + numAvailableBytes = Base.kMatchMaxLen; + + UInt32 repMaxIndex = 0; + UInt32 i; + for (i = 0; i < Base.kNumRepDistances; i++) + { + reps[i] = _repDistances[i]; + repLens[i] = _matchFinder.GetMatchLen(0 - 1, reps[i], Base.kMatchMaxLen); + if (repLens[i] > repLens[repMaxIndex]) + repMaxIndex = i; + } + if (repLens[repMaxIndex] >= _numFastBytes) + { + backRes = repMaxIndex; + UInt32 lenRes = repLens[repMaxIndex]; + MovePos(lenRes - 1); + return lenRes; + } + + if (lenMain >= _numFastBytes) + { + backRes = _matchDistances[numDistancePairs - 1] + Base.kNumRepDistances; + MovePos(lenMain - 1); + return lenMain; + } + + Byte currentByte = _matchFinder.GetIndexByte(0 - 1); + Byte matchByte = _matchFinder.GetIndexByte((Int32)(0 - _repDistances[0] - 1 - 1)); + + if (lenMain < 2 && currentByte != matchByte && repLens[repMaxIndex] < 2) + { + backRes = (UInt32)0xFFFFFFFF; + return 1; + } + + _optimum[0].State = _state; + + UInt32 posState = (position & _posStateMask); + + _optimum[1].Price = _isMatch[(_state.Index << Base.kNumPosStatesBitsMax) + posState].GetPrice0() + + _literalEncoder.GetSubCoder(position, _previousByte).GetPrice(!_state.IsCharState(), matchByte, currentByte); + _optimum[1].MakeAsChar(); + + UInt32 matchPrice = _isMatch[(_state.Index << Base.kNumPosStatesBitsMax) + posState].GetPrice1(); + UInt32 repMatchPrice = matchPrice + _isRep[_state.Index].GetPrice1(); + + if (matchByte == currentByte) + { + UInt32 shortRepPrice = repMatchPrice + GetRepLen1Price(_state, posState); + if (shortRepPrice < _optimum[1].Price) + { + _optimum[1].Price = shortRepPrice; + _optimum[1].MakeAsShortRep(); + } + } + + UInt32 lenEnd = ((lenMain >= repLens[repMaxIndex]) ? lenMain : repLens[repMaxIndex]); + + if(lenEnd < 2) + { + backRes = _optimum[1].BackPrev; + return 1; + } + + _optimum[1].PosPrev = 0; + + _optimum[0].Backs0 = reps[0]; + _optimum[0].Backs1 = reps[1]; + _optimum[0].Backs2 = reps[2]; + _optimum[0].Backs3 = reps[3]; + + UInt32 len = lenEnd; + do + _optimum[len--].Price = kIfinityPrice; + while (len >= 2); + + for (i = 0; i < Base.kNumRepDistances; i++) + { + UInt32 repLen = repLens[i]; + if (repLen < 2) + continue; + UInt32 price = repMatchPrice + GetPureRepPrice(i, _state, posState); + do + { + UInt32 curAndLenPrice = price + _repMatchLenEncoder.GetPrice(repLen - 2, posState); + Optimal optimum = _optimum[repLen]; + if (curAndLenPrice < optimum.Price) + { + optimum.Price = curAndLenPrice; + optimum.PosPrev = 0; + optimum.BackPrev = i; + optimum.Prev1IsChar = false; + } + } + while (--repLen >= 2); + } + + UInt32 normalMatchPrice = matchPrice + _isRep[_state.Index].GetPrice0(); + + len = ((repLens[0] >= 2) ? repLens[0] + 1 : 2); + if (len <= lenMain) + { + UInt32 offs = 0; + while (len > _matchDistances[offs]) + offs += 2; + for (; ; len++) + { + UInt32 distance = _matchDistances[offs + 1]; + UInt32 curAndLenPrice = normalMatchPrice + GetPosLenPrice(distance, len, posState); + Optimal optimum = _optimum[len]; + if (curAndLenPrice < optimum.Price) + { + optimum.Price = curAndLenPrice; + optimum.PosPrev = 0; + optimum.BackPrev = distance + Base.kNumRepDistances; + optimum.Prev1IsChar = false; + } + if (len == _matchDistances[offs]) + { + offs += 2; + if (offs == numDistancePairs) + break; + } + } + } + + UInt32 cur = 0; + + while (true) + { + cur++; + if (cur == lenEnd) + return Backward(out backRes, cur); + UInt32 newLen; + ReadMatchDistances(out newLen, out numDistancePairs); + if (newLen >= _numFastBytes) + { + _numDistancePairs = numDistancePairs; + _longestMatchLength = newLen; + _longestMatchWasFound = true; + return Backward(out backRes, cur); + } + position++; + UInt32 posPrev = _optimum[cur].PosPrev; + Base.State state; + if (_optimum[cur].Prev1IsChar) + { + posPrev--; + if (_optimum[cur].Prev2) + { + state = _optimum[_optimum[cur].PosPrev2].State; + if (_optimum[cur].BackPrev2 < Base.kNumRepDistances) + state.UpdateRep(); + else + state.UpdateMatch(); + } + else + state = _optimum[posPrev].State; + state.UpdateChar(); + } + else + state = _optimum[posPrev].State; + if (posPrev == cur - 1) + { + if (_optimum[cur].IsShortRep()) + state.UpdateShortRep(); + else + state.UpdateChar(); + } + else + { + UInt32 pos; + if (_optimum[cur].Prev1IsChar && _optimum[cur].Prev2) + { + posPrev = _optimum[cur].PosPrev2; + pos = _optimum[cur].BackPrev2; + state.UpdateRep(); + } + else + { + pos = _optimum[cur].BackPrev; + if (pos < Base.kNumRepDistances) + state.UpdateRep(); + else + state.UpdateMatch(); + } + Optimal opt = _optimum[posPrev]; + if (pos < Base.kNumRepDistances) + { + if (pos == 0) + { + reps[0] = opt.Backs0; + reps[1] = opt.Backs1; + reps[2] = opt.Backs2; + reps[3] = opt.Backs3; + } + else if (pos == 1) + { + reps[0] = opt.Backs1; + reps[1] = opt.Backs0; + reps[2] = opt.Backs2; + reps[3] = opt.Backs3; + } + else if (pos == 2) + { + reps[0] = opt.Backs2; + reps[1] = opt.Backs0; + reps[2] = opt.Backs1; + reps[3] = opt.Backs3; + } + else + { + reps[0] = opt.Backs3; + reps[1] = opt.Backs0; + reps[2] = opt.Backs1; + reps[3] = opt.Backs2; + } + } + else + { + reps[0] = (pos - Base.kNumRepDistances); + reps[1] = opt.Backs0; + reps[2] = opt.Backs1; + reps[3] = opt.Backs2; + } + } + _optimum[cur].State = state; + _optimum[cur].Backs0 = reps[0]; + _optimum[cur].Backs1 = reps[1]; + _optimum[cur].Backs2 = reps[2]; + _optimum[cur].Backs3 = reps[3]; + UInt32 curPrice = _optimum[cur].Price; + + currentByte = _matchFinder.GetIndexByte(0 - 1); + matchByte = _matchFinder.GetIndexByte((Int32)(0 - reps[0] - 1 - 1)); + + posState = (position & _posStateMask); + + UInt32 curAnd1Price = curPrice + + _isMatch[(state.Index << Base.kNumPosStatesBitsMax) + posState].GetPrice0() + + _literalEncoder.GetSubCoder(position, _matchFinder.GetIndexByte(0 - 2)). + GetPrice(!state.IsCharState(), matchByte, currentByte); + + Optimal nextOptimum = _optimum[cur + 1]; + + bool nextIsChar = false; + if (curAnd1Price < nextOptimum.Price) + { + nextOptimum.Price = curAnd1Price; + nextOptimum.PosPrev = cur; + nextOptimum.MakeAsChar(); + nextIsChar = true; + } + + matchPrice = curPrice + _isMatch[(state.Index << Base.kNumPosStatesBitsMax) + posState].GetPrice1(); + repMatchPrice = matchPrice + _isRep[state.Index].GetPrice1(); + + if (matchByte == currentByte && + !(nextOptimum.PosPrev < cur && nextOptimum.BackPrev == 0)) + { + UInt32 shortRepPrice = repMatchPrice + GetRepLen1Price(state, posState); + if (shortRepPrice <= nextOptimum.Price) + { + nextOptimum.Price = shortRepPrice; + nextOptimum.PosPrev = cur; + nextOptimum.MakeAsShortRep(); + nextIsChar = true; + } + } + + UInt32 numAvailableBytesFull = _matchFinder.GetNumAvailableBytes() + 1; + numAvailableBytesFull = Math.Min(kNumOpts - 1 - cur, numAvailableBytesFull); + numAvailableBytes = numAvailableBytesFull; + + if (numAvailableBytes < 2) + continue; + if (numAvailableBytes > _numFastBytes) + numAvailableBytes = _numFastBytes; + if (!nextIsChar && matchByte != currentByte) + { + // try Literal + rep0 + UInt32 t = Math.Min(numAvailableBytesFull - 1, _numFastBytes); + UInt32 lenTest2 = _matchFinder.GetMatchLen(0, reps[0], t); + if (lenTest2 >= 2) + { + Base.State state2 = state; + state2.UpdateChar(); + UInt32 posStateNext = (position + 1) & _posStateMask; + UInt32 nextRepMatchPrice = curAnd1Price + + _isMatch[(state2.Index << Base.kNumPosStatesBitsMax) + posStateNext].GetPrice1() + + _isRep[state2.Index].GetPrice1(); + { + UInt32 offset = cur + 1 + lenTest2; + while (lenEnd < offset) + _optimum[++lenEnd].Price = kIfinityPrice; + UInt32 curAndLenPrice = nextRepMatchPrice + GetRepPrice( + 0, lenTest2, state2, posStateNext); + Optimal optimum = _optimum[offset]; + if (curAndLenPrice < optimum.Price) + { + optimum.Price = curAndLenPrice; + optimum.PosPrev = cur + 1; + optimum.BackPrev = 0; + optimum.Prev1IsChar = true; + optimum.Prev2 = false; + } + } + } + } + + UInt32 startLen = 2; // speed optimization + + for (UInt32 repIndex = 0; repIndex < Base.kNumRepDistances; repIndex++) + { + UInt32 lenTest = _matchFinder.GetMatchLen(0 - 1, reps[repIndex], numAvailableBytes); + if (lenTest < 2) + continue; + UInt32 lenTestTemp = lenTest; + do + { + while (lenEnd < cur + lenTest) + _optimum[++lenEnd].Price = kIfinityPrice; + UInt32 curAndLenPrice = repMatchPrice + GetRepPrice(repIndex, lenTest, state, posState); + Optimal optimum = _optimum[cur + lenTest]; + if (curAndLenPrice < optimum.Price) + { + optimum.Price = curAndLenPrice; + optimum.PosPrev = cur; + optimum.BackPrev = repIndex; + optimum.Prev1IsChar = false; + } + } + while(--lenTest >= 2); + lenTest = lenTestTemp; + + if (repIndex == 0) + startLen = lenTest + 1; + + // if (_maxMode) + if (lenTest < numAvailableBytesFull) + { + UInt32 t = Math.Min(numAvailableBytesFull - 1 - lenTest, _numFastBytes); + UInt32 lenTest2 = _matchFinder.GetMatchLen((Int32)lenTest, reps[repIndex], t); + if (lenTest2 >= 2) + { + Base.State state2 = state; + state2.UpdateRep(); + UInt32 posStateNext = (position + lenTest) & _posStateMask; + UInt32 curAndLenCharPrice = + repMatchPrice + GetRepPrice(repIndex, lenTest, state, posState) + + _isMatch[(state2.Index << Base.kNumPosStatesBitsMax) + posStateNext].GetPrice0() + + _literalEncoder.GetSubCoder(position + lenTest, + _matchFinder.GetIndexByte((Int32)lenTest - 1 - 1)).GetPrice(true, + _matchFinder.GetIndexByte((Int32)((Int32)lenTest - 1 - (Int32)(reps[repIndex] + 1))), + _matchFinder.GetIndexByte((Int32)lenTest - 1)); + state2.UpdateChar(); + posStateNext = (position + lenTest + 1) & _posStateMask; + UInt32 nextMatchPrice = curAndLenCharPrice + _isMatch[(state2.Index << Base.kNumPosStatesBitsMax) + posStateNext].GetPrice1(); + UInt32 nextRepMatchPrice = nextMatchPrice + _isRep[state2.Index].GetPrice1(); + + // for(; lenTest2 >= 2; lenTest2--) + { + UInt32 offset = lenTest + 1 + lenTest2; + while(lenEnd < cur + offset) + _optimum[++lenEnd].Price = kIfinityPrice; + UInt32 curAndLenPrice = nextRepMatchPrice + GetRepPrice(0, lenTest2, state2, posStateNext); + Optimal optimum = _optimum[cur + offset]; + if (curAndLenPrice < optimum.Price) + { + optimum.Price = curAndLenPrice; + optimum.PosPrev = cur + lenTest + 1; + optimum.BackPrev = 0; + optimum.Prev1IsChar = true; + optimum.Prev2 = true; + optimum.PosPrev2 = cur; + optimum.BackPrev2 = repIndex; + } + } + } + } + } + + if (newLen > numAvailableBytes) + { + newLen = numAvailableBytes; + for (numDistancePairs = 0; newLen > _matchDistances[numDistancePairs]; numDistancePairs += 2) ; + _matchDistances[numDistancePairs] = newLen; + numDistancePairs += 2; + } + if (newLen >= startLen) + { + normalMatchPrice = matchPrice + _isRep[state.Index].GetPrice0(); + while (lenEnd < cur + newLen) + _optimum[++lenEnd].Price = kIfinityPrice; + + UInt32 offs = 0; + while (startLen > _matchDistances[offs]) + offs += 2; + + for (UInt32 lenTest = startLen; ; lenTest++) + { + UInt32 curBack = _matchDistances[offs + 1]; + UInt32 curAndLenPrice = normalMatchPrice + GetPosLenPrice(curBack, lenTest, posState); + Optimal optimum = _optimum[cur + lenTest]; + if (curAndLenPrice < optimum.Price) + { + optimum.Price = curAndLenPrice; + optimum.PosPrev = cur; + optimum.BackPrev = curBack + Base.kNumRepDistances; + optimum.Prev1IsChar = false; + } + + if (lenTest == _matchDistances[offs]) + { + if (lenTest < numAvailableBytesFull) + { + UInt32 t = Math.Min(numAvailableBytesFull - 1 - lenTest, _numFastBytes); + UInt32 lenTest2 = _matchFinder.GetMatchLen((Int32)lenTest, curBack, t); + if (lenTest2 >= 2) + { + Base.State state2 = state; + state2.UpdateMatch(); + UInt32 posStateNext = (position + lenTest) & _posStateMask; + UInt32 curAndLenCharPrice = curAndLenPrice + + _isMatch[(state2.Index << Base.kNumPosStatesBitsMax) + posStateNext].GetPrice0() + + _literalEncoder.GetSubCoder(position + lenTest, + _matchFinder.GetIndexByte((Int32)lenTest - 1 - 1)). + GetPrice(true, + _matchFinder.GetIndexByte((Int32)lenTest - (Int32)(curBack + 1) - 1), + _matchFinder.GetIndexByte((Int32)lenTest - 1)); + state2.UpdateChar(); + posStateNext = (position + lenTest + 1) & _posStateMask; + UInt32 nextMatchPrice = curAndLenCharPrice + _isMatch[(state2.Index << Base.kNumPosStatesBitsMax) + posStateNext].GetPrice1(); + UInt32 nextRepMatchPrice = nextMatchPrice + _isRep[state2.Index].GetPrice1(); + + UInt32 offset = lenTest + 1 + lenTest2; + while (lenEnd < cur + offset) + _optimum[++lenEnd].Price = kIfinityPrice; + curAndLenPrice = nextRepMatchPrice + GetRepPrice(0, lenTest2, state2, posStateNext); + optimum = _optimum[cur + offset]; + if (curAndLenPrice < optimum.Price) + { + optimum.Price = curAndLenPrice; + optimum.PosPrev = cur + lenTest + 1; + optimum.BackPrev = 0; + optimum.Prev1IsChar = true; + optimum.Prev2 = true; + optimum.PosPrev2 = cur; + optimum.BackPrev2 = curBack + Base.kNumRepDistances; + } + } + } + offs += 2; + if (offs == numDistancePairs) + break; + } + } + } + } + } + + bool ChangePair(UInt32 smallDist, UInt32 bigDist) + { + const int kDif = 7; + return (smallDist < ((UInt32)(1) << (32 - kDif)) && bigDist >= (smallDist << kDif)); + } + + void WriteEndMarker(UInt32 posState) + { + if (!_writeEndMark) + return; + + _isMatch[(_state.Index << Base.kNumPosStatesBitsMax) + posState].Encode(_rangeEncoder, 1); + _isRep[_state.Index].Encode(_rangeEncoder, 0); + _state.UpdateMatch(); + UInt32 len = Base.kMatchMinLen; + _lenEncoder.Encode(_rangeEncoder, len - Base.kMatchMinLen, posState); + UInt32 posSlot = (1 << Base.kNumPosSlotBits) - 1; + UInt32 lenToPosState = Base.GetLenToPosState(len); + _posSlotEncoder[lenToPosState].Encode(_rangeEncoder, posSlot); + int footerBits = 30; + UInt32 posReduced = (((UInt32)1) << footerBits) - 1; + _rangeEncoder.EncodeDirectBits(posReduced >> Base.kNumAlignBits, footerBits - Base.kNumAlignBits); + _posAlignEncoder.ReverseEncode(_rangeEncoder, posReduced & Base.kAlignMask); + } + + void Flush(UInt32 nowPos) + { + ReleaseMFStream(); + WriteEndMarker(nowPos & _posStateMask); + _rangeEncoder.FlushData(); + _rangeEncoder.FlushStream(); + } + + public void CodeOneBlock(out Int64 inSize, out Int64 outSize, out bool finished) + { + inSize = 0; + outSize = 0; + finished = true; + + if (_inStream != null) + { + _matchFinder.SetStream(_inStream); + _matchFinder.Init(); + _needReleaseMFStream = true; + _inStream = null; + if (_trainSize > 0) + _matchFinder.Skip(_trainSize); + } + + if (_finished) + return; + _finished = true; + + + Int64 progressPosValuePrev = nowPos64; + if (nowPos64 == 0) + { + if (_matchFinder.GetNumAvailableBytes() == 0) + { + Flush((UInt32)nowPos64); + return; + } + UInt32 len, numDistancePairs; // it's not used + ReadMatchDistances(out len, out numDistancePairs); + UInt32 posState = (UInt32)(nowPos64) & _posStateMask; + _isMatch[(_state.Index << Base.kNumPosStatesBitsMax) + posState].Encode(_rangeEncoder, 0); + _state.UpdateChar(); + Byte curByte = _matchFinder.GetIndexByte((Int32)(0 - _additionalOffset)); + _literalEncoder.GetSubCoder((UInt32)(nowPos64), _previousByte).Encode(_rangeEncoder, curByte); + _previousByte = curByte; + _additionalOffset--; + nowPos64++; + } + if (_matchFinder.GetNumAvailableBytes() == 0) + { + Flush((UInt32)nowPos64); + return; + } + while (true) + { + UInt32 pos; + UInt32 len = GetOptimum((UInt32)nowPos64, out pos); + + UInt32 posState = ((UInt32)nowPos64) & _posStateMask; + UInt32 complexState = (_state.Index << Base.kNumPosStatesBitsMax) + posState; + if (len == 1 && pos == 0xFFFFFFFF) + { + _isMatch[complexState].Encode(_rangeEncoder, 0); + Byte curByte = _matchFinder.GetIndexByte((Int32)(0 - _additionalOffset)); + LiteralEncoder.Encoder2 subCoder = _literalEncoder.GetSubCoder((UInt32)nowPos64, _previousByte); + if (!_state.IsCharState()) + { + Byte matchByte = _matchFinder.GetIndexByte((Int32)(0 - _repDistances[0] - 1 - _additionalOffset)); + subCoder.EncodeMatched(_rangeEncoder, matchByte, curByte); + } + else + subCoder.Encode(_rangeEncoder, curByte); + _previousByte = curByte; + _state.UpdateChar(); + } + else + { + _isMatch[complexState].Encode(_rangeEncoder, 1); + if (pos < Base.kNumRepDistances) + { + _isRep[_state.Index].Encode(_rangeEncoder, 1); + if (pos == 0) + { + _isRepG0[_state.Index].Encode(_rangeEncoder, 0); + if (len == 1) + _isRep0Long[complexState].Encode(_rangeEncoder, 0); + else + _isRep0Long[complexState].Encode(_rangeEncoder, 1); + } + else + { + _isRepG0[_state.Index].Encode(_rangeEncoder, 1); + if (pos == 1) + _isRepG1[_state.Index].Encode(_rangeEncoder, 0); + else + { + _isRepG1[_state.Index].Encode(_rangeEncoder, 1); + _isRepG2[_state.Index].Encode(_rangeEncoder, pos - 2); + } + } + if (len == 1) + _state.UpdateShortRep(); + else + { + _repMatchLenEncoder.Encode(_rangeEncoder, len - Base.kMatchMinLen, posState); + _state.UpdateRep(); + } + UInt32 distance = _repDistances[pos]; + if (pos != 0) + { + for (UInt32 i = pos; i >= 1; i--) + _repDistances[i] = _repDistances[i - 1]; + _repDistances[0] = distance; + } + } + else + { + _isRep[_state.Index].Encode(_rangeEncoder, 0); + _state.UpdateMatch(); + _lenEncoder.Encode(_rangeEncoder, len - Base.kMatchMinLen, posState); + pos -= Base.kNumRepDistances; + UInt32 posSlot = GetPosSlot(pos); + UInt32 lenToPosState = Base.GetLenToPosState(len); + _posSlotEncoder[lenToPosState].Encode(_rangeEncoder, posSlot); + + if (posSlot >= Base.kStartPosModelIndex) + { + int footerBits = (int)((posSlot >> 1) - 1); + UInt32 baseVal = ((2 | (posSlot & 1)) << footerBits); + UInt32 posReduced = pos - baseVal; + + if (posSlot < Base.kEndPosModelIndex) + RangeCoder.BitTreeEncoder.ReverseEncode(_posEncoders, + baseVal - posSlot - 1, _rangeEncoder, footerBits, posReduced); + else + { + _rangeEncoder.EncodeDirectBits(posReduced >> Base.kNumAlignBits, footerBits - Base.kNumAlignBits); + _posAlignEncoder.ReverseEncode(_rangeEncoder, posReduced & Base.kAlignMask); + _alignPriceCount++; + } + } + UInt32 distance = pos; + for (UInt32 i = Base.kNumRepDistances - 1; i >= 1; i--) + _repDistances[i] = _repDistances[i - 1]; + _repDistances[0] = distance; + _matchPriceCount++; + } + _previousByte = _matchFinder.GetIndexByte((Int32)(len - 1 - _additionalOffset)); + } + _additionalOffset -= len; + nowPos64 += len; + if (_additionalOffset == 0) + { + // if (!_fastMode) + if (_matchPriceCount >= (1 << 7)) + FillDistancesPrices(); + if (_alignPriceCount >= Base.kAlignTableSize) + FillAlignPrices(); + inSize = nowPos64; + outSize = _rangeEncoder.GetProcessedSizeAdd(); + if (_matchFinder.GetNumAvailableBytes() == 0) + { + Flush((UInt32)nowPos64); + return; + } + + if (nowPos64 - progressPosValuePrev >= (1 << 12)) + { + _finished = false; + finished = false; + return; + } + } + } + } + + void ReleaseMFStream() + { + if (_matchFinder != null && _needReleaseMFStream) + { + _matchFinder.ReleaseStream(); + _needReleaseMFStream = false; + } + } + + void SetOutStream(System.IO.Stream outStream) { _rangeEncoder.SetStream(outStream); } + void ReleaseOutStream() { _rangeEncoder.ReleaseStream(); } + + void ReleaseStreams() + { + ReleaseMFStream(); + ReleaseOutStream(); + } + + void SetStreams(System.IO.Stream inStream, System.IO.Stream outStream, + Int64 inSize, Int64 outSize) + { + _inStream = inStream; + _finished = false; + Create(); + SetOutStream(outStream); + Init(); + + // if (!_fastMode) + { + FillDistancesPrices(); + FillAlignPrices(); + } + + _lenEncoder.SetTableSize(_numFastBytes + 1 - Base.kMatchMinLen); + _lenEncoder.UpdateTables((UInt32)1 << _posStateBits); + _repMatchLenEncoder.SetTableSize(_numFastBytes + 1 - Base.kMatchMinLen); + _repMatchLenEncoder.UpdateTables((UInt32)1 << _posStateBits); + + nowPos64 = 0; + } + + + public void Code(System.IO.Stream inStream, System.IO.Stream outStream, + Int64 inSize, Int64 outSize, ICodeProgress progress) + { + _needReleaseMFStream = false; + try + { + SetStreams(inStream, outStream, inSize, outSize); + while (true) + { + Int64 processedInSize; + Int64 processedOutSize; + bool finished; + CodeOneBlock(out processedInSize, out processedOutSize, out finished); + if (finished) + return; + if (progress != null) + { + progress.SetProgress(processedInSize, processedOutSize); + } + } + } + finally + { + ReleaseStreams(); + } + } + + const int kPropSize = 5; + Byte[] properties = new Byte[kPropSize]; + + public void WriteCoderProperties(System.IO.Stream outStream) + { + properties[0] = (Byte)((_posStateBits * 5 + _numLiteralPosStateBits) * 9 + _numLiteralContextBits); + for (int i = 0; i < 4; i++) + properties[1 + i] = (Byte)((_dictionarySize >> (8 * i)) & 0xFF); + outStream.Write(properties, 0, kPropSize); + } + + UInt32[] tempPrices = new UInt32[Base.kNumFullDistances]; + UInt32 _matchPriceCount; + + void FillDistancesPrices() + { + for (UInt32 i = Base.kStartPosModelIndex; i < Base.kNumFullDistances; i++) + { + UInt32 posSlot = GetPosSlot(i); + int footerBits = (int)((posSlot >> 1) - 1); + UInt32 baseVal = ((2 | (posSlot & 1)) << footerBits); + tempPrices[i] = BitTreeEncoder.ReverseGetPrice(_posEncoders, + baseVal - posSlot - 1, footerBits, i - baseVal); + } + + for (UInt32 lenToPosState = 0; lenToPosState < Base.kNumLenToPosStates; lenToPosState++) + { + UInt32 posSlot; + RangeCoder.BitTreeEncoder encoder = _posSlotEncoder[lenToPosState]; + + UInt32 st = (lenToPosState << Base.kNumPosSlotBits); + for (posSlot = 0; posSlot < _distTableSize; posSlot++) + _posSlotPrices[st + posSlot] = encoder.GetPrice(posSlot); + for (posSlot = Base.kEndPosModelIndex; posSlot < _distTableSize; posSlot++) + _posSlotPrices[st + posSlot] += ((((posSlot >> 1) - 1) - Base.kNumAlignBits) << RangeCoder.BitEncoder.kNumBitPriceShiftBits); + + UInt32 st2 = lenToPosState * Base.kNumFullDistances; + UInt32 i; + for (i = 0; i < Base.kStartPosModelIndex; i++) + _distancesPrices[st2 + i] = _posSlotPrices[st + i]; + for (; i < Base.kNumFullDistances; i++) + _distancesPrices[st2 + i] = _posSlotPrices[st + GetPosSlot(i)] + tempPrices[i]; + } + _matchPriceCount = 0; + } + + void FillAlignPrices() + { + for (UInt32 i = 0; i < Base.kAlignTableSize; i++) + _alignPrices[i] = _posAlignEncoder.ReverseGetPrice(i); + _alignPriceCount = 0; + } + + + static string[] kMatchFinderIDs = + { + "BT2", + "BT4", + }; + + static int FindMatchFinder(string s) + { + for (int m = 0; m < kMatchFinderIDs.Length; m++) + if (s == kMatchFinderIDs[m]) + return m; + return -1; + } + + public void SetCoderProperties(CoderPropID[] propIDs, object[] properties) + { + for (UInt32 i = 0; i < properties.Length; i++) + { + object prop = properties[i]; + switch (propIDs[i]) + { + case CoderPropID.NumFastBytes: + { + if (!(prop is Int32)) + throw new InvalidParamException(); + Int32 numFastBytes = (Int32)prop; + if (numFastBytes < 5 || numFastBytes > Base.kMatchMaxLen) + throw new InvalidParamException(); + _numFastBytes = (UInt32)numFastBytes; + break; + } + case CoderPropID.Algorithm: + { + /* + if (!(prop is Int32)) + throw new InvalidParamException(); + Int32 maximize = (Int32)prop; + _fastMode = (maximize == 0); + _maxMode = (maximize >= 2); + */ + break; + } + case CoderPropID.MatchFinder: + { + if (!(prop is String)) + throw new InvalidParamException(); + EMatchFinderType matchFinderIndexPrev = _matchFinderType; + int m = FindMatchFinder(((string)prop).ToUpper()); + if (m < 0) + throw new InvalidParamException(); + _matchFinderType = (EMatchFinderType)m; + if (_matchFinder != null && matchFinderIndexPrev != _matchFinderType) + { + _dictionarySizePrev = 0xFFFFFFFF; + _matchFinder = null; + } + break; + } + case CoderPropID.DictionarySize: + { + const int kDicLogSizeMaxCompress = 30; + if (!(prop is Int32)) + throw new InvalidParamException(); ; + Int32 dictionarySize = (Int32)prop; + if (dictionarySize < (UInt32)(1 << Base.kDicLogSizeMin) || + dictionarySize > (UInt32)(1 << kDicLogSizeMaxCompress)) + throw new InvalidParamException(); + _dictionarySize = (UInt32)dictionarySize; + int dicLogSize; + for (dicLogSize = 0; dicLogSize < (UInt32)kDicLogSizeMaxCompress; dicLogSize++) + if (dictionarySize <= ((UInt32)(1) << dicLogSize)) + break; + _distTableSize = (UInt32)dicLogSize * 2; + break; + } + case CoderPropID.PosStateBits: + { + if (!(prop is Int32)) + throw new InvalidParamException(); + Int32 v = (Int32)prop; + if (v < 0 || v > (UInt32)Base.kNumPosStatesBitsEncodingMax) + throw new InvalidParamException(); + _posStateBits = (int)v; + _posStateMask = (((UInt32)1) << (int)_posStateBits) - 1; + break; + } + case CoderPropID.LitPosBits: + { + if (!(prop is Int32)) + throw new InvalidParamException(); + Int32 v = (Int32)prop; + if (v < 0 || v > (UInt32)Base.kNumLitPosStatesBitsEncodingMax) + throw new InvalidParamException(); + _numLiteralPosStateBits = (int)v; + break; + } + case CoderPropID.LitContextBits: + { + if (!(prop is Int32)) + throw new InvalidParamException(); + Int32 v = (Int32)prop; + if (v < 0 || v > (UInt32)Base.kNumLitContextBitsMax) + throw new InvalidParamException(); ; + _numLiteralContextBits = (int)v; + break; + } + case CoderPropID.EndMarker: + { + if (!(prop is Boolean)) + throw new InvalidParamException(); + SetWriteEndMarkerMode((Boolean)prop); + break; + } + default: + throw new InvalidParamException(); + } + } + } + + uint _trainSize = 0; + public void SetTrainSize(uint trainSize) + { + _trainSize = trainSize; + } + + } +} diff --git a/src/Microsoft.DotNet.Archive/LZMA/Compress/RangeCoder/RangeCoder.cs b/src/Microsoft.DotNet.Archive/LZMA/Compress/RangeCoder/RangeCoder.cs new file mode 100644 index 0000000000..d9c2e30ce0 --- /dev/null +++ b/src/Microsoft.DotNet.Archive/LZMA/Compress/RangeCoder/RangeCoder.cs @@ -0,0 +1,237 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; + +namespace SevenZip.Compression.RangeCoder +{ + class Encoder + { + public const uint kTopValue = (1 << 24); + + System.IO.Stream Stream; + + public UInt64 Low; + public uint Range; + uint _cacheSize; + byte _cache; + + long StartPosition; + + public void SetStream(System.IO.Stream stream) + { + Stream = stream; + } + + public void ReleaseStream() + { + Stream = null; + } + + public void Init() + { + StartPosition = Stream.Position; + + Low = 0; + Range = 0xFFFFFFFF; + _cacheSize = 1; + _cache = 0; + } + + public void FlushData() + { + for (int i = 0; i < 5; i++) + ShiftLow(); + } + + public void FlushStream() + { + Stream.Flush(); + } + + public void CloseStream() + { + Stream.Dispose(); + } + + public void Encode(uint start, uint size, uint total) + { + Low += start * (Range /= total); + Range *= size; + while (Range < kTopValue) + { + Range <<= 8; + ShiftLow(); + } + } + + public void ShiftLow() + { + if ((uint)Low < (uint)0xFF000000 || (uint)(Low >> 32) == 1) + { + byte temp = _cache; + do + { + Stream.WriteByte((byte)(temp + (Low >> 32))); + temp = 0xFF; + } + while (--_cacheSize != 0); + _cache = (byte)(((uint)Low) >> 24); + } + _cacheSize++; + Low = ((uint)Low) << 8; + } + + public void EncodeDirectBits(uint v, int numTotalBits) + { + for (int i = numTotalBits - 1; i >= 0; i--) + { + Range >>= 1; + if (((v >> i) & 1) == 1) + Low += Range; + if (Range < kTopValue) + { + Range <<= 8; + ShiftLow(); + } + } + } + + public void EncodeBit(uint size0, int numTotalBits, uint symbol) + { + uint newBound = (Range >> numTotalBits) * size0; + if (symbol == 0) + Range = newBound; + else + { + Low += newBound; + Range -= newBound; + } + while (Range < kTopValue) + { + Range <<= 8; + ShiftLow(); + } + } + + public long GetProcessedSizeAdd() + { + return _cacheSize + + Stream.Position - StartPosition + 4; + // (long)Stream.GetProcessedSize(); + } + } + + class Decoder + { + public const uint kTopValue = (1 << 24); + public uint Range; + public uint Code; + // public Buffer.InBuffer Stream = new Buffer.InBuffer(1 << 16); + public System.IO.Stream Stream; + + public void Init(System.IO.Stream stream) + { + // Stream.Init(stream); + Stream = stream; + + Code = 0; + Range = 0xFFFFFFFF; + for (int i = 0; i < 5; i++) + Code = (Code << 8) | (byte)Stream.ReadByte(); + } + + public void ReleaseStream() + { + // Stream.ReleaseStream(); + Stream = null; + } + + public void CloseStream() + { + Stream.Dispose(); + } + + public void Normalize() + { + while (Range < kTopValue) + { + Code = (Code << 8) | (byte)Stream.ReadByte(); + Range <<= 8; + } + } + + public void Normalize2() + { + if (Range < kTopValue) + { + Code = (Code << 8) | (byte)Stream.ReadByte(); + Range <<= 8; + } + } + + public uint GetThreshold(uint total) + { + return Code / (Range /= total); + } + + public void Decode(uint start, uint size, uint total) + { + Code -= start * Range; + Range *= size; + Normalize(); + } + + public uint DecodeDirectBits(int numTotalBits) + { + uint range = Range; + uint code = Code; + uint result = 0; + for (int i = numTotalBits; i > 0; i--) + { + range >>= 1; + /* + result <<= 1; + if (code >= range) + { + code -= range; + result |= 1; + } + */ + uint t = (code - range) >> 31; + code -= range & (t - 1); + result = (result << 1) | (1 - t); + + if (range < kTopValue) + { + code = (code << 8) | (byte)Stream.ReadByte(); + range <<= 8; + } + } + Range = range; + Code = code; + return result; + } + + public uint DecodeBit(uint size0, int numTotalBits) + { + uint newBound = (Range >> numTotalBits) * size0; + uint symbol; + if (Code < newBound) + { + symbol = 0; + Range = newBound; + } + else + { + symbol = 1; + Code -= newBound; + Range -= newBound; + } + Normalize(); + return symbol; + } + + // ulong GetProcessedSize() {return Stream.GetProcessedSize(); } + } +} diff --git a/src/Microsoft.DotNet.Archive/LZMA/Compress/RangeCoder/RangeCoderBit.cs b/src/Microsoft.DotNet.Archive/LZMA/Compress/RangeCoder/RangeCoderBit.cs new file mode 100644 index 0000000000..46d27ed0f6 --- /dev/null +++ b/src/Microsoft.DotNet.Archive/LZMA/Compress/RangeCoder/RangeCoderBit.cs @@ -0,0 +1,120 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; + +namespace SevenZip.Compression.RangeCoder +{ + struct BitEncoder + { + public const int kNumBitModelTotalBits = 11; + public const uint kBitModelTotal = (1 << kNumBitModelTotalBits); + const int kNumMoveBits = 5; + const int kNumMoveReducingBits = 2; + public const int kNumBitPriceShiftBits = 6; + + uint Prob; + + public void Init() { Prob = kBitModelTotal >> 1; } + + public void UpdateModel(uint symbol) + { + if (symbol == 0) + Prob += (kBitModelTotal - Prob) >> kNumMoveBits; + else + Prob -= (Prob) >> kNumMoveBits; + } + + public void Encode(Encoder encoder, uint symbol) + { + // encoder.EncodeBit(Prob, kNumBitModelTotalBits, symbol); + // UpdateModel(symbol); + uint newBound = (encoder.Range >> kNumBitModelTotalBits) * Prob; + if (symbol == 0) + { + encoder.Range = newBound; + Prob += (kBitModelTotal - Prob) >> kNumMoveBits; + } + else + { + encoder.Low += newBound; + encoder.Range -= newBound; + Prob -= (Prob) >> kNumMoveBits; + } + if (encoder.Range < Encoder.kTopValue) + { + encoder.Range <<= 8; + encoder.ShiftLow(); + } + } + + private static UInt32[] ProbPrices = new UInt32[kBitModelTotal >> kNumMoveReducingBits]; + + static BitEncoder() + { + const int kNumBits = (kNumBitModelTotalBits - kNumMoveReducingBits); + for (int i = kNumBits - 1; i >= 0; i--) + { + UInt32 start = (UInt32)1 << (kNumBits - i - 1); + UInt32 end = (UInt32)1 << (kNumBits - i); + for (UInt32 j = start; j < end; j++) + ProbPrices[j] = ((UInt32)i << kNumBitPriceShiftBits) + + (((end - j) << kNumBitPriceShiftBits) >> (kNumBits - i - 1)); + } + } + + public uint GetPrice(uint symbol) + { + return ProbPrices[(((Prob - symbol) ^ ((-(int)symbol))) & (kBitModelTotal - 1)) >> kNumMoveReducingBits]; + } + public uint GetPrice0() { return ProbPrices[Prob >> kNumMoveReducingBits]; } + public uint GetPrice1() { return ProbPrices[(kBitModelTotal - Prob) >> kNumMoveReducingBits]; } + } + + struct BitDecoder + { + public const int kNumBitModelTotalBits = 11; + public const uint kBitModelTotal = (1 << kNumBitModelTotalBits); + const int kNumMoveBits = 5; + + uint Prob; + + public void UpdateModel(int numMoveBits, uint symbol) + { + if (symbol == 0) + Prob += (kBitModelTotal - Prob) >> numMoveBits; + else + Prob -= (Prob) >> numMoveBits; + } + + public void Init() { Prob = kBitModelTotal >> 1; } + + public uint Decode(RangeCoder.Decoder rangeDecoder) + { + uint newBound = (uint)(rangeDecoder.Range >> kNumBitModelTotalBits) * (uint)Prob; + if (rangeDecoder.Code < newBound) + { + rangeDecoder.Range = newBound; + Prob += (kBitModelTotal - Prob) >> kNumMoveBits; + if (rangeDecoder.Range < Decoder.kTopValue) + { + rangeDecoder.Code = (rangeDecoder.Code << 8) | (byte)rangeDecoder.Stream.ReadByte(); + rangeDecoder.Range <<= 8; + } + return 0; + } + else + { + rangeDecoder.Range -= newBound; + rangeDecoder.Code -= newBound; + Prob -= (Prob) >> kNumMoveBits; + if (rangeDecoder.Range < Decoder.kTopValue) + { + rangeDecoder.Code = (rangeDecoder.Code << 8) | (byte)rangeDecoder.Stream.ReadByte(); + rangeDecoder.Range <<= 8; + } + return 1; + } + } + } +} diff --git a/src/Microsoft.DotNet.Archive/LZMA/Compress/RangeCoder/RangeCoderBitTree.cs b/src/Microsoft.DotNet.Archive/LZMA/Compress/RangeCoder/RangeCoderBitTree.cs new file mode 100644 index 0000000000..f7985c47ba --- /dev/null +++ b/src/Microsoft.DotNet.Archive/LZMA/Compress/RangeCoder/RangeCoderBitTree.cs @@ -0,0 +1,160 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; + +namespace SevenZip.Compression.RangeCoder +{ + struct BitTreeEncoder + { + BitEncoder[] Models; + int NumBitLevels; + + public BitTreeEncoder(int numBitLevels) + { + NumBitLevels = numBitLevels; + Models = new BitEncoder[1 << numBitLevels]; + } + + public void Init() + { + for (uint i = 1; i < (1 << NumBitLevels); i++) + Models[i].Init(); + } + + public void Encode(Encoder rangeEncoder, UInt32 symbol) + { + UInt32 m = 1; + for (int bitIndex = NumBitLevels; bitIndex > 0; ) + { + bitIndex--; + UInt32 bit = (symbol >> bitIndex) & 1; + Models[m].Encode(rangeEncoder, bit); + m = (m << 1) | bit; + } + } + + public void ReverseEncode(Encoder rangeEncoder, UInt32 symbol) + { + UInt32 m = 1; + for (UInt32 i = 0; i < NumBitLevels; i++) + { + UInt32 bit = symbol & 1; + Models[m].Encode(rangeEncoder, bit); + m = (m << 1) | bit; + symbol >>= 1; + } + } + + public UInt32 GetPrice(UInt32 symbol) + { + UInt32 price = 0; + UInt32 m = 1; + for (int bitIndex = NumBitLevels; bitIndex > 0; ) + { + bitIndex--; + UInt32 bit = (symbol >> bitIndex) & 1; + price += Models[m].GetPrice(bit); + m = (m << 1) + bit; + } + return price; + } + + public UInt32 ReverseGetPrice(UInt32 symbol) + { + UInt32 price = 0; + UInt32 m = 1; + for (int i = NumBitLevels; i > 0; i--) + { + UInt32 bit = symbol & 1; + symbol >>= 1; + price += Models[m].GetPrice(bit); + m = (m << 1) | bit; + } + return price; + } + + public static UInt32 ReverseGetPrice(BitEncoder[] Models, UInt32 startIndex, + int NumBitLevels, UInt32 symbol) + { + UInt32 price = 0; + UInt32 m = 1; + for (int i = NumBitLevels; i > 0; i--) + { + UInt32 bit = symbol & 1; + symbol >>= 1; + price += Models[startIndex + m].GetPrice(bit); + m = (m << 1) | bit; + } + return price; + } + + public static void ReverseEncode(BitEncoder[] Models, UInt32 startIndex, + Encoder rangeEncoder, int NumBitLevels, UInt32 symbol) + { + UInt32 m = 1; + for (int i = 0; i < NumBitLevels; i++) + { + UInt32 bit = symbol & 1; + Models[startIndex + m].Encode(rangeEncoder, bit); + m = (m << 1) | bit; + symbol >>= 1; + } + } + } + + struct BitTreeDecoder + { + BitDecoder[] Models; + int NumBitLevels; + + public BitTreeDecoder(int numBitLevels) + { + NumBitLevels = numBitLevels; + Models = new BitDecoder[1 << numBitLevels]; + } + + public void Init() + { + for (uint i = 1; i < (1 << NumBitLevels); i++) + Models[i].Init(); + } + + public uint Decode(RangeCoder.Decoder rangeDecoder) + { + uint m = 1; + for (int bitIndex = NumBitLevels; bitIndex > 0; bitIndex--) + m = (m << 1) + Models[m].Decode(rangeDecoder); + return m - ((uint)1 << NumBitLevels); + } + + public uint ReverseDecode(RangeCoder.Decoder rangeDecoder) + { + uint m = 1; + uint symbol = 0; + for (int bitIndex = 0; bitIndex < NumBitLevels; bitIndex++) + { + uint bit = Models[m].Decode(rangeDecoder); + m <<= 1; + m += bit; + symbol |= (bit << bitIndex); + } + return symbol; + } + + public static uint ReverseDecode(BitDecoder[] Models, UInt32 startIndex, + RangeCoder.Decoder rangeDecoder, int NumBitLevels) + { + uint m = 1; + uint symbol = 0; + for (int bitIndex = 0; bitIndex < NumBitLevels; bitIndex++) + { + uint bit = Models[startIndex + m].Decode(rangeDecoder); + m <<= 1; + m += bit; + symbol |= (bit << bitIndex); + } + return symbol; + } + } +} diff --git a/src/Microsoft.DotNet.Archive/LZMA/ICoder.cs b/src/Microsoft.DotNet.Archive/LZMA/ICoder.cs new file mode 100644 index 0000000000..992f6823fb --- /dev/null +++ b/src/Microsoft.DotNet.Archive/LZMA/ICoder.cs @@ -0,0 +1,160 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +// ICoder.h + +using System; + +namespace SevenZip +{ + /// + /// The exception that is thrown when an error in input stream occurs during decoding. + /// + class DataErrorException : Exception + { + public DataErrorException(): base("Data Error") { } + } + + /// + /// The exception that is thrown when the value of an argument is outside the allowable range. + /// + class InvalidParamException : Exception + { + public InvalidParamException(): base("Invalid Parameter") { } + } + + public interface ICodeProgress + { + /// + /// Callback progress. + /// + /// + /// input size. -1 if unknown. + /// + /// + /// output size. -1 if unknown. + /// + void SetProgress(Int64 inSize, Int64 outSize); + }; + + public interface ICoder + { + /// + /// Codes streams. + /// + /// + /// input Stream. + /// + /// + /// output Stream. + /// + /// + /// input Size. -1 if unknown. + /// + /// + /// output Size. -1 if unknown. + /// + /// + /// callback progress reference. + /// + /// + /// if input stream is not valid + /// + void Code(System.IO.Stream inStream, System.IO.Stream outStream, + Int64 inSize, Int64 outSize, ICodeProgress progress); + }; + + /* + public interface ICoder2 + { + void Code(ISequentialInStream []inStreams, + const UInt64 []inSizes, + ISequentialOutStream []outStreams, + UInt64 []outSizes, + ICodeProgress progress); + }; + */ + + /// + /// Provides the fields that represent properties idenitifiers for compressing. + /// + public enum CoderPropID + { + /// + /// Specifies default property. + /// + DefaultProp = 0, + /// + /// Specifies size of dictionary. + /// + DictionarySize, + /// + /// Specifies size of memory for PPM*. + /// + UsedMemorySize, + /// + /// Specifies order for PPM methods. + /// + Order, + /// + /// Specifies Block Size. + /// + BlockSize, + /// + /// Specifies number of postion state bits for LZMA (0 <= x <= 4). + /// + PosStateBits, + /// + /// Specifies number of literal context bits for LZMA (0 <= x <= 8). + /// + LitContextBits, + /// + /// Specifies number of literal position bits for LZMA (0 <= x <= 4). + /// + LitPosBits, + /// + /// Specifies number of fast bytes for LZ*. + /// + NumFastBytes, + /// + /// Specifies match finder. LZMA: "BT2", "BT4" or "BT4B". + /// + MatchFinder, + /// + /// Specifies the number of match finder cyckes. + /// + MatchFinderCycles, + /// + /// Specifies number of passes. + /// + NumPasses, + /// + /// Specifies number of algorithm. + /// + Algorithm, + /// + /// Specifies the number of threads. + /// + NumThreads, + /// + /// Specifies mode with end marker. + /// + EndMarker + }; + + + public interface ISetCoderProperties + { + void SetCoderProperties(CoderPropID[] propIDs, object[] properties); + }; + + public interface IWriteCoderProperties + { + void WriteCoderProperties(System.IO.Stream outStream); + } + + public interface ISetDecoderProperties + { + void SetDecoderProperties(byte[] properties); + } +} diff --git a/src/Microsoft.DotNet.Archive/LZMA/README.md b/src/Microsoft.DotNet.Archive/LZMA/README.md new file mode 100644 index 0000000000..74e2758529 --- /dev/null +++ b/src/Microsoft.DotNet.Archive/LZMA/README.md @@ -0,0 +1,10 @@ +## LZMA SDK +This source came from the C# implementation of LZMA from the LZMA SDK, version 16.02, from http://www.7-zip.org/sdk.html. + +## License +LZMA SDK is placed in the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or distribute the original LZMA SDK code, either in source code form or as a compiled binary, for any purpose, commercial or non-commercial, and by any means. + +## Thanks! +Thanks goes to Igor Pavlov for making this available. diff --git a/src/Microsoft.DotNet.Archive/Microsoft.DotNet.Archive.csproj b/src/Microsoft.DotNet.Archive/Microsoft.DotNet.Archive.csproj new file mode 100644 index 0000000000..bb2c678962 --- /dev/null +++ b/src/Microsoft.DotNet.Archive/Microsoft.DotNet.Archive.csproj @@ -0,0 +1,14 @@ + + + + + + netstandard1.3 + false + + + + + + + diff --git a/src/Microsoft.DotNet.Archive/ProgressReport.cs b/src/Microsoft.DotNet.Archive/ProgressReport.cs new file mode 100644 index 0000000000..6eb9555633 --- /dev/null +++ b/src/Microsoft.DotNet.Archive/ProgressReport.cs @@ -0,0 +1,29 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; + +namespace Microsoft.DotNet.Archive +{ + public struct ProgressReport + { + public ProgressReport(string phase, long ticks, long total) + { + Phase = phase; + Ticks = ticks; + Total = total; + } + public string Phase { get; } + public long Ticks { get; } + public long Total { get; } + } + + public static class ProgressReportExtensions + { + public static void Report(this IProgress progress, string phase, long ticks, long total) + { + progress.Report(new ProgressReport(phase, ticks, total)); + } + } + +} diff --git a/src/Microsoft.DotNet.Archive/Properties/AssemblyInfo.cs b/src/Microsoft.DotNet.Archive/Properties/AssemblyInfo.cs new file mode 100644 index 0000000000..110018e7fd --- /dev/null +++ b/src/Microsoft.DotNet.Archive/Properties/AssemblyInfo.cs @@ -0,0 +1,6 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Reflection; + +[assembly: AssemblyMetadataAttribute("Serviceable", "True")] diff --git a/src/Microsoft.DotNet.Archive/ThreadLocalZipArchive.cs b/src/Microsoft.DotNet.Archive/ThreadLocalZipArchive.cs new file mode 100644 index 0000000000..7f61207cb5 --- /dev/null +++ b/src/Microsoft.DotNet.Archive/ThreadLocalZipArchive.cs @@ -0,0 +1,59 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.IO; +using System.IO.Compression; +using System.Threading; + +namespace Microsoft.DotNet.Archive +{ + /// + /// Wraps ThreadLocal and exposes Dispose semantics that dispose all archives + /// + internal class ThreadLocalZipArchive : IDisposable + { + private ThreadLocal _archive; + private bool _disposed = false; + + public ThreadLocalZipArchive(string archivePath, ZipArchive local = null) + { + _archive = new ThreadLocal(() => + new ZipArchive(File.Open(archivePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete), ZipArchiveMode.Read), + trackAllValues:true); + + if (local != null) + { + // reuse provided one for current thread + _archive.Value = local; + } + } + + public ZipArchive Archive { get { return _archive.Value; } } + + public void Dispose() + { + if (!_disposed) + { + if (_archive != null) + { + // dispose all archives + if (_archive.Values != null) + { + foreach (var value in _archive.Values) + { + if (value != null) + { + value.Dispose(); + } + } + } + + // dispose ThreadLocal + _archive.Dispose(); + _archive = null; + } + } + } + } +} diff --git a/src/dotnet-archive/CommandLine/CommandArgument.cs b/src/dotnet-archive/CommandLine/CommandArgument.cs new file mode 100644 index 0000000000..045609d79f --- /dev/null +++ b/src/dotnet-archive/CommandLine/CommandArgument.cs @@ -0,0 +1,29 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.DotNet.Cli.CommandLine +{ + internal class CommandArgument + { + public CommandArgument() + { + Values = new List(); + } + + public string Name { get; set; } + public string Description { get; set; } + public List Values { get; private set; } + public bool MultipleValues { get; set; } + public string Value + { + get + { + return Values.FirstOrDefault(); + } + } + } +} diff --git a/src/dotnet-archive/CommandLine/CommandLineApplication.cs b/src/dotnet-archive/CommandLine/CommandLineApplication.cs new file mode 100644 index 0000000000..64b6b3d17c --- /dev/null +++ b/src/dotnet-archive/CommandLine/CommandLineApplication.cs @@ -0,0 +1,693 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.DotNet.Cli.CommandLine +{ + internal class CommandLineApplication + { + private enum ParseOptionResult + { + Succeeded, + ShowHelp, + ShowVersion, + UnexpectedArgs, + } + + // Indicates whether the parser should throw an exception when it runs into an unexpected argument. + // If this field is set to false, the parser will stop parsing when it sees an unexpected argument, and all + // remaining arguments, including the first unexpected argument, will be stored in RemainingArguments property. + private readonly bool _throwOnUnexpectedArg; + + public CommandLineApplication(bool throwOnUnexpectedArg = true) + { + _throwOnUnexpectedArg = throwOnUnexpectedArg; + Options = new List(); + Arguments = new List(); + Commands = new List(); + RemainingArguments = new List(); + Invoke = () => 0; + } + + public CommandLineApplication Parent { get; set; } + public string Name { get; set; } + public string FullName { get; set; } + public string Syntax { get; set; } + public string Description { get; set; } + public List Options { get; private set; } + public CommandOption OptionHelp { get; private set; } + public CommandOption OptionVersion { get; private set; } + public List Arguments { get; private set; } + public List RemainingArguments { get; private set; } + public bool IsShowingInformation { get; protected set; } // Is showing help or version? + public Func Invoke { get; set; } + public Func LongVersionGetter { get; set; } + public Func ShortVersionGetter { get; set; } + public List Commands { get; private set; } + public bool HandleResponseFiles { get; set; } + public bool AllowArgumentSeparator { get; set; } + public bool HandleRemainingArguments { get; set; } + public string ArgumentSeparatorHelpText { get; set; } + + public CommandLineApplication AddCommand(string name, bool throwOnUnexpectedArg = true) + { + return AddCommand(name, _ => { }, throwOnUnexpectedArg); + } + + public CommandLineApplication AddCommand(string name, Action configuration, + bool throwOnUnexpectedArg = true) + { + var command = new CommandLineApplication(throwOnUnexpectedArg) { Name = name }; + return AddCommand(command, configuration, throwOnUnexpectedArg); + } + + public CommandLineApplication AddCommand(CommandLineApplication command, bool throwOnUnexpectedArg = true) + { + return AddCommand(command, _ => { }, throwOnUnexpectedArg); + } + + public CommandLineApplication AddCommand( + CommandLineApplication command, + Action configuration, + bool throwOnUnexpectedArg = true) + { + if (command == null || configuration == null) + { + throw new NullReferenceException(); + } + + command.Parent = this; + Commands.Add(command); + configuration(command); + return command; + } + + public CommandOption Option(string template, string description, CommandOptionType optionType) + { + return Option(template, description, optionType, _ => { }); + } + + public CommandOption Option(string template, string description, CommandOptionType optionType, Action configuration) + { + var option = new CommandOption(template, optionType) { Description = description }; + Options.Add(option); + configuration(option); + return option; + } + + public CommandArgument Argument(string name, string description, bool multipleValues = false) + { + return Argument(name, description, _ => { }, multipleValues); + } + + public CommandArgument Argument(string name, string description, Action configuration, bool multipleValues = false) + { + var lastArg = Arguments.LastOrDefault(); + if (lastArg != null && lastArg.MultipleValues) + { + var message = string.Format(LocalizableStrings.LastArgumentMultiValueError, + lastArg.Name); + throw new InvalidOperationException(message); + } + + var argument = new CommandArgument { Name = name, Description = description, MultipleValues = multipleValues }; + Arguments.Add(argument); + configuration(argument); + return argument; + } + + public void OnExecute(Func invoke) + { + Invoke = invoke; + } + + public void OnExecute(Func> invoke) + { + Invoke = () => invoke().Result; + } + + public int Execute(params string[] args) + { + CommandLineApplication command = this; + CommandArgumentEnumerator arguments = null; + + if (HandleResponseFiles) + { + args = ExpandResponseFiles(args).ToArray(); + } + + for (var index = 0; index < args.Length; index++) + { + var arg = args[index]; + + bool isLongOption = arg.StartsWith("--"); + if (arg == "-?" || arg == "/?") + { + command.ShowHelp(); + return 0; + } + else if (isLongOption || arg.StartsWith("-")) + { + CommandOption option; + + var result = ParseOption(isLongOption, command, args, ref index, out option); + + + if (result == ParseOptionResult.ShowHelp) + { + command.ShowHelp(); + return 0; + } + else if (result == ParseOptionResult.ShowVersion) + { + command.ShowVersion(); + return 0; + } + else if (result == ParseOptionResult.UnexpectedArgs) + { + break; + } + } + else + { + var subcommand = ParseSubCommand(arg, command); + if (subcommand != null) + { + command = subcommand; + } + else + { + if (arguments == null || arguments.CommandName != command.Name) + { + arguments = new CommandArgumentEnumerator(command.Arguments.GetEnumerator(), command.Name); + } + + if (arguments.MoveNext()) + { + arguments.Current.Values.Add(arg); + } + else + { + HandleUnexpectedArg(command, args, index, argTypeName: "command or argument"); + break; + } + } + } + } + + if (Commands.Count > 0 && command == this) + { + throw new CommandParsingException( + command, + "Required command missing", + isRequiredSubCommandMissing: true); + } + + return command.Invoke(); + } + + private ParseOptionResult ParseOption( + bool isLongOption, + CommandLineApplication command, + string[] args, + ref int index, + out CommandOption option) + { + option = null; + ParseOptionResult result = ParseOptionResult.Succeeded; + var arg = args[index]; + + int optionPrefixLength = isLongOption ? 2 : 1; + string[] optionComponents = arg.Substring(optionPrefixLength).Split(new[] { ':', '=' }, 2); + string optionName = optionComponents[0]; + + if (isLongOption) + { + option = command.Options.SingleOrDefault( + opt => string.Equals(opt.LongName, optionName, StringComparison.Ordinal)); + } + else + { + option = command.Options.SingleOrDefault( + opt => string.Equals(opt.ShortName, optionName, StringComparison.Ordinal)); + + if (option == null) + { + option = command.Options.SingleOrDefault( + opt => string.Equals(opt.SymbolName, optionName, StringComparison.Ordinal)); + } + } + + if (option == null) + { + if (isLongOption && string.IsNullOrEmpty(optionName) && + !command._throwOnUnexpectedArg && AllowArgumentSeparator) + { + // a stand-alone "--" is the argument separator, so skip it and + // handle the rest of the args as unexpected args + index++; + } + + HandleUnexpectedArg(command, args, index, argTypeName: "option"); + result = ParseOptionResult.UnexpectedArgs; + } + else if (command.OptionHelp == option) + { + result = ParseOptionResult.ShowHelp; + } + else if (command.OptionVersion == option) + { + result = ParseOptionResult.ShowVersion; + } + else + { + if (optionComponents.Length == 2) + { + if (!option.TryParse(optionComponents[1])) + { + command.ShowHint(); + throw new CommandParsingException(command, + String.Format(LocalizableStrings.UnexpectedValueForOptionError, optionComponents[1], optionName)); + } + } + else + { + if (option.OptionType == CommandOptionType.NoValue || + option.OptionType == CommandOptionType.BoolValue) + { + // No value is needed for this option + option.TryParse(null); + } + else + { + index++; + + if (index < args.Length) + { + arg = args[index]; + if (!option.TryParse(arg)) + { + command.ShowHint(); + throw new CommandParsingException( + command, + String.Format(LocalizableStrings.UnexpectedValueForOptionError, arg, optionName)); + } + } + else + { + command.ShowHint(); + throw new CommandParsingException( + command, + String.Format(LocalizableStrings.OptionRequiresSingleValueWhichIsMissing, arg, optionName)); + } + } + } + } + + return result; + } + + private CommandLineApplication ParseSubCommand(string arg, CommandLineApplication command) + { + foreach (var subcommand in command.Commands) + { + if (string.Equals(subcommand.Name, arg, StringComparison.OrdinalIgnoreCase)) + { + return subcommand; + } + } + + return null; + } + + // Helper method that adds a help option + public CommandOption HelpOption(string template) + { + // Help option is special because we stop parsing once we see it + // So we store it separately for further use + OptionHelp = Option(template, LocalizableStrings.ShowHelpInfo, CommandOptionType.NoValue); + + return OptionHelp; + } + + public CommandOption VersionOption(string template, + string shortFormVersion, + string longFormVersion = null) + { + if (longFormVersion == null) + { + return VersionOption(template, () => shortFormVersion); + } + else + { + return VersionOption(template, () => shortFormVersion, () => longFormVersion); + } + } + + // Helper method that adds a version option + public CommandOption VersionOption(string template, + Func shortFormVersionGetter, + Func longFormVersionGetter = null) + { + // Version option is special because we stop parsing once we see it + // So we store it separately for further use + OptionVersion = Option(template, LocalizableStrings.ShowVersionInfo, CommandOptionType.NoValue); + ShortVersionGetter = shortFormVersionGetter; + LongVersionGetter = longFormVersionGetter ?? shortFormVersionGetter; + + return OptionVersion; + } + + // Show short hint that reminds users to use help option + public void ShowHint() + { + if (OptionHelp != null) + { + Console.WriteLine(string.Format(LocalizableStrings.ShowHintInfo, OptionHelp.LongName)); + } + } + + // Show full help + public void ShowHelp(string commandName = null) + { + var headerBuilder = new StringBuilder(LocalizableStrings.UsageHeader); + var usagePrefixLength = headerBuilder.Length; + for (var cmd = this; cmd != null; cmd = cmd.Parent) + { + cmd.IsShowingInformation = true; + if (cmd != this && cmd.Arguments.Any()) + { + var args = string.Join(" ", cmd.Arguments.Select(arg => arg.Name)); + headerBuilder.Insert(usagePrefixLength, string.Format(" {0} {1}", cmd.Name, args)); + } + else + { + headerBuilder.Insert(usagePrefixLength, string.Format(" {0}", cmd.Name)); + } + } + + CommandLineApplication target; + + if (commandName == null || string.Equals(Name, commandName, StringComparison.OrdinalIgnoreCase)) + { + target = this; + } + else + { + target = Commands.SingleOrDefault(cmd => string.Equals(cmd.Name, commandName, StringComparison.OrdinalIgnoreCase)); + + if (target != null) + { + headerBuilder.AppendFormat(" {0}", commandName); + } + else + { + // The command name is invalid so don't try to show help for something that doesn't exist + target = this; + } + + } + + var optionsBuilder = new StringBuilder(); + var commandsBuilder = new StringBuilder(); + var argumentsBuilder = new StringBuilder(); + var argumentSeparatorBuilder = new StringBuilder(); + + int maxArgLen = 0; + for (var cmd = target; cmd != null; cmd = cmd.Parent) + { + if (cmd.Arguments.Any()) + { + if (cmd == target) + { + headerBuilder.Append(LocalizableStrings.UsageArgumentsToken); + } + + if (argumentsBuilder.Length == 0) + { + argumentsBuilder.AppendLine(); + argumentsBuilder.AppendLine(LocalizableStrings.UsageArgumentsHeader); + } + + maxArgLen = Math.Max(maxArgLen, MaxArgumentLength(cmd.Arguments)); + } + } + + for (var cmd = target; cmd != null; cmd = cmd.Parent) + { + if (cmd.Arguments.Any()) + { + foreach (var arg in cmd.Arguments) + { + argumentsBuilder.AppendFormat( + " {0}{1}", + arg.Name.PadRight(maxArgLen + 2), + arg.Description); + argumentsBuilder.AppendLine(); + } + } + } + + if (target.Options.Any()) + { + headerBuilder.Append(LocalizableStrings.UsageOptionsToken); + + optionsBuilder.AppendLine(); + optionsBuilder.AppendLine(LocalizableStrings.UsageOptionsHeader); + var maxOptLen = MaxOptionTemplateLength(target.Options); + var outputFormat = string.Format(" {{0, -{0}}}{{1}}", maxOptLen + 2); + foreach (var opt in target.Options) + { + optionsBuilder.AppendFormat(outputFormat, opt.Template, opt.Description); + optionsBuilder.AppendLine(); + } + } + + if (target.Commands.Any()) + { + headerBuilder.Append(LocalizableStrings.UsageCommandToken); + + commandsBuilder.AppendLine(); + commandsBuilder.AppendLine(LocalizableStrings.UsageCommandsHeader); + var maxCmdLen = MaxCommandLength(target.Commands); + var outputFormat = string.Format(" {{0, -{0}}}{{1}}", maxCmdLen + 2); + foreach (var cmd in target.Commands.OrderBy(c => c.Name)) + { + commandsBuilder.AppendFormat(outputFormat, cmd.Name, cmd.Description); + commandsBuilder.AppendLine(); + } + + if (OptionHelp != null) + { + commandsBuilder.AppendLine(); + commandsBuilder.AppendFormat(LocalizableStrings.UsageCommandsDetailHelp, Name); + commandsBuilder.AppendLine(); + } + } + + if (target.AllowArgumentSeparator || target.HandleRemainingArguments) + { + if (target.AllowArgumentSeparator) + { + headerBuilder.Append(LocalizableStrings.UsageCommandAdditionalArgs); + } + else + { + headerBuilder.Append(LocalizableStrings.UsageCommandArgs); + } + + if (!string.IsNullOrEmpty(target.ArgumentSeparatorHelpText)) + { + argumentSeparatorBuilder.AppendLine(); + argumentSeparatorBuilder.AppendLine(LocalizableStrings.UsageCommandsAdditionalArgsHeader); + argumentSeparatorBuilder.AppendLine(String.Format(" {0}", target.ArgumentSeparatorHelpText)); + argumentSeparatorBuilder.AppendLine(); + } + } + + headerBuilder.AppendLine(); + + var nameAndVersion = new StringBuilder(); + nameAndVersion.AppendLine(GetFullNameAndVersion()); + nameAndVersion.AppendLine(); + + Console.Write("{0}{1}{2}{3}{4}{5}", nameAndVersion, headerBuilder, argumentsBuilder, optionsBuilder, commandsBuilder, argumentSeparatorBuilder); + } + + public void ShowVersion() + { + for (var cmd = this; cmd != null; cmd = cmd.Parent) + { + cmd.IsShowingInformation = true; + } + + Console.WriteLine(FullName); + Console.WriteLine(LongVersionGetter()); + } + + public string GetFullNameAndVersion() + { + return ShortVersionGetter == null ? FullName : string.Format("{0} {1}", FullName, ShortVersionGetter()); + } + + public void ShowRootCommandFullNameAndVersion() + { + var rootCmd = this; + while (rootCmd.Parent != null) + { + rootCmd = rootCmd.Parent; + } + + Console.WriteLine(rootCmd.GetFullNameAndVersion()); + Console.WriteLine(); + } + + private int MaxOptionTemplateLength(IEnumerable options) + { + var maxLen = 0; + foreach (var opt in options) + { + maxLen = opt.Template.Length > maxLen ? opt.Template.Length : maxLen; + } + return maxLen; + } + + private int MaxCommandLength(IEnumerable commands) + { + var maxLen = 0; + foreach (var cmd in commands) + { + maxLen = cmd.Name.Length > maxLen ? cmd.Name.Length : maxLen; + } + return maxLen; + } + + private int MaxArgumentLength(IEnumerable arguments) + { + var maxLen = 0; + foreach (var arg in arguments) + { + maxLen = arg.Name.Length > maxLen ? arg.Name.Length : maxLen; + } + return maxLen; + } + + private void HandleUnexpectedArg(CommandLineApplication command, string[] args, int index, string argTypeName) + { + if (command._throwOnUnexpectedArg) + { + command.ShowHint(); + throw new CommandParsingException(command, String.Format(LocalizableStrings.UnexpectedArgumentError, argTypeName, args[index])); + } + else + { + // All remaining arguments are stored for further use + command.RemainingArguments.AddRange(new ArraySegment(args, index, args.Length - index)); + } + } + + private IEnumerable ExpandResponseFiles(IEnumerable args) + { + foreach (var arg in args) + { + if (!arg.StartsWith("@", StringComparison.Ordinal)) + { + yield return arg; + } + else + { + var fileName = arg.Substring(1); + + var responseFileArguments = ParseResponseFile(fileName); + + // ParseResponseFile can suppress expanding this response file by + // returning null. In that case, we'll treat the response + // file token as a regular argument. + + if (responseFileArguments == null) + { + yield return arg; + } + else + { + foreach (var responseFileArgument in responseFileArguments) + yield return responseFileArgument.Trim(); + } + } + } + } + + private IEnumerable ParseResponseFile(string fileName) + { + if (!HandleResponseFiles) + return null; + + if (!File.Exists(fileName)) + { + throw new InvalidOperationException(String.Format(LocalizableStrings.ResponseFileNotFoundError, fileName)); + } + + return File.ReadLines(fileName); + } + + private class CommandArgumentEnumerator : IEnumerator + { + private readonly IEnumerator _enumerator; + + public CommandArgumentEnumerator( + IEnumerator enumerator, + string commandName) + { + CommandName = commandName; + _enumerator = enumerator; + } + + public string CommandName { get; } + + public CommandArgument Current + { + get + { + return _enumerator.Current; + } + } + + object IEnumerator.Current + { + get + { + return Current; + } + } + + public void Dispose() + { + _enumerator.Dispose(); + } + + public bool MoveNext() + { + if (Current == null || !Current.MultipleValues) + { + return _enumerator.MoveNext(); + } + + // If current argument allows multiple values, we don't move forward and + // all later values will be added to current CommandArgument.Values + return true; + } + + public void Reset() + { + _enumerator.Reset(); + } + } + } +} diff --git a/src/dotnet-archive/CommandLine/CommandOption.cs b/src/dotnet-archive/CommandLine/CommandOption.cs new file mode 100644 index 0000000000..caed0bd779 --- /dev/null +++ b/src/dotnet-archive/CommandLine/CommandOption.cs @@ -0,0 +1,135 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.DotNet.Cli.CommandLine +{ + internal class CommandOption + { + public CommandOption(string template, CommandOptionType optionType) + { + Template = template; + OptionType = optionType; + Values = new List(); + + foreach (var part in Template.Split(new[] { ' ', '|' }, StringSplitOptions.RemoveEmptyEntries)) + { + if (part.StartsWith("--")) + { + LongName = part.Substring(2); + } + else if (part.StartsWith("-")) + { + var optName = part.Substring(1); + + // If there is only one char and it is not an English letter, it is a symbol option (e.g. "-?") + if (optName.Length == 1 && !IsEnglishLetter(optName[0])) + { + SymbolName = optName; + } + else + { + ShortName = optName; + } + } + else if (part.StartsWith("<") && part.EndsWith(">")) + { + ValueName = part.Substring(1, part.Length - 2); + } + else if (optionType == CommandOptionType.MultipleValue && part.StartsWith("<") && part.EndsWith(">...")) + { + ValueName = part.Substring(1, part.Length - 5); + } + else + { + throw new ArgumentException(String.Format(LocalizableStrings.InvalidTemplateError, nameof(template))); + } + } + + if (string.IsNullOrEmpty(LongName) && string.IsNullOrEmpty(ShortName) && string.IsNullOrEmpty(SymbolName)) + { + throw new ArgumentException(LocalizableStrings.InvalidTemplateError, nameof(template)); + } + } + + public string Template { get; set; } + public string ShortName { get; set; } + public string LongName { get; set; } + public string SymbolName { get; set; } + public string ValueName { get; set; } + public string Description { get; set; } + public List Values { get; private set; } + public bool? BoolValue { get; private set; } + public CommandOptionType OptionType { get; private set; } + + public bool TryParse(string value) + { + switch (OptionType) + { + case CommandOptionType.MultipleValue: + Values.Add(value); + break; + case CommandOptionType.SingleValue: + if (Values.Any()) + { + return false; + } + Values.Add(value); + break; + case CommandOptionType.BoolValue: + if (Values.Any()) + { + return false; + } + + if (value == null) + { + // add null to indicate that the option was present, but had no value + Values.Add(null); + BoolValue = true; + } + else + { + bool boolValue; + if (!bool.TryParse(value, out boolValue)) + { + return false; + } + + Values.Add(value); + BoolValue = boolValue; + } + break; + case CommandOptionType.NoValue: + if (value != null) + { + return false; + } + // Add a value to indicate that this option was specified + Values.Add("on"); + break; + default: + break; + } + return true; + } + + public bool HasValue() + { + return Values.Any(); + } + + public string Value() + { + return HasValue() ? Values[0] : null; + } + + private bool IsEnglishLetter(char c) + { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); + } + } +} diff --git a/src/dotnet-archive/CommandLine/CommandOptionType.cs b/src/dotnet-archive/CommandLine/CommandOptionType.cs new file mode 100644 index 0000000000..6cee7406b7 --- /dev/null +++ b/src/dotnet-archive/CommandLine/CommandOptionType.cs @@ -0,0 +1,14 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + + +namespace Microsoft.DotNet.Cli.CommandLine +{ + internal enum CommandOptionType + { + MultipleValue, + SingleValue, + BoolValue, + NoValue + } +} diff --git a/src/dotnet-archive/CommandLine/CommandParsingException.cs b/src/dotnet-archive/CommandLine/CommandParsingException.cs new file mode 100644 index 0000000000..82c675f4bb --- /dev/null +++ b/src/dotnet-archive/CommandLine/CommandParsingException.cs @@ -0,0 +1,45 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using Microsoft.DotNet.Tools; + +namespace Microsoft.DotNet.Cli.CommandLine +{ + internal class CommandParsingException : Exception + { + private readonly bool _isRequiredSubCommandMissing; + + public CommandParsingException( + string message, + string helpText = null) : base(message) + { + HelpText = helpText ?? ""; + Data.Add("CLI_User_Displayed_Exception", true); + } + + public CommandParsingException( + CommandLineApplication command, + string message, + bool isRequiredSubCommandMissing = false) + : this(message) + { + Command = command; + _isRequiredSubCommandMissing = isRequiredSubCommandMissing; + } + + public CommandLineApplication Command { get; } + + public string HelpText { get; } = ""; + + public override string Message + { + get + { + return _isRequiredSubCommandMissing + ? CommonLocalizableStrings.RequiredCommandNotPassed + : base.Message; + } + } + } +} \ No newline at end of file diff --git a/src/dotnet-archive/CommandLine/HelpMessageStrings.cs b/src/dotnet-archive/CommandLine/HelpMessageStrings.cs new file mode 100644 index 0000000000..09551d2e66 --- /dev/null +++ b/src/dotnet-archive/CommandLine/HelpMessageStrings.cs @@ -0,0 +1,10 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.DotNet.Cli.CommandLine +{ + internal class HelpMessageStrings + { + internal const string MSBuildAdditionalArgsHelpText = LocalizableStrings.MSBuildAdditionalArgsHelpText; + } +} diff --git a/src/dotnet-archive/CommandLine/LocalizableStrings.cs b/src/dotnet-archive/CommandLine/LocalizableStrings.cs new file mode 100644 index 0000000000..ff01edf1be --- /dev/null +++ b/src/dotnet-archive/CommandLine/LocalizableStrings.cs @@ -0,0 +1,50 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.DotNet.Cli.CommandLine +{ + internal class LocalizableStrings + { + public const string LastArgumentMultiValueError = "The last argument '{0}' accepts multiple values. No more argument can be added."; + + public const string OptionRequiresSingleValueWhichIsMissing = "Required value for option '{0}' was not provided."; + + public const string UnexpectedValueForOptionError = "Unexpected value '{0}' for option '{1}'"; + + public const string UnexpectedArgumentError = "Unrecognized {0} '{1}'"; + + public const string ResponseFileNotFoundError = "Response file '{0}' doesn't exist."; + + public const string ShowHelpInfo = "Show help information"; + + public const string ShowVersionInfo = "Show version information"; + + public const string ShowHintInfo = "Specify --{0} for a list of available options and commands."; + + public const string UsageHeader = "Usage:"; + + public const string UsageArgumentsToken = " [arguments]"; + + public const string UsageArgumentsHeader = "Arguments:"; + + public const string UsageOptionsToken = " [options]"; + + public const string UsageOptionsHeader = "Options:"; + + public const string UsageCommandToken = " [command]"; + + public const string UsageCommandsHeader = "Commands:"; + + public const string UsageCommandsDetailHelp = "Use \"{0} [command] --help\" for more information about a command."; + + public const string UsageCommandArgs = " [args]"; + + public const string UsageCommandAdditionalArgs = " [[--] ...]]"; + + public const string UsageCommandsAdditionalArgsHeader = "Additional Arguments:"; + + public const string InvalidTemplateError = "Invalid template pattern '{0}'"; + + public const string MSBuildAdditionalArgsHelpText = "Any extra options that should be passed to MSBuild. See 'dotnet msbuild -h' for available options."; + } +} diff --git a/src/dotnet-archive/CommonLocalizableStrings.cs b/src/dotnet-archive/CommonLocalizableStrings.cs new file mode 100644 index 0000000000..d19b462d55 --- /dev/null +++ b/src/dotnet-archive/CommonLocalizableStrings.cs @@ -0,0 +1,189 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.DotNet.Tools +{ + internal class CommonLocalizableStrings + { + public const string UnsupportedProjectType = "Unsupported project type. Please check with your sdk provider."; + public const string ProjectAlreadyHasAreference = "Project already has a reference to `{0}`."; + public const string ProjectReferenceCouldNotBeFound = "Project reference `{0}` could not be found."; + public const string ProjectReferenceRemoved = "Project reference `{0}` removed."; + + // Project related + public const string Project = "Project"; + public const string ProjectFile = "Project file"; + public const string Reference = "Reference"; + public const string ProjectReference = "Project reference"; + public const string ProjectReferenceOneOrMore = "Project reference(s)"; + public const string PackageReference = "Package reference"; + public const string P2P = "Project to Project"; + public const string P2PReference = "Project to Project reference"; + public const string Package = "Package"; + public const string Solution = "Solution"; + public const string SolutionFile = "Solution file"; + public const string Executable = "Executable"; + public const string Library = "Library"; + public const string Program = "Program"; + public const string Application = "Application"; + public const string ReferenceAddedToTheProject = "Reference `{0}` added to the project."; + + // Verbs + public const string Add = "Add"; + public const string Remove = "Remove"; + public const string Delete = "Delete"; + public const string Update = "Update"; + public const string New = "New"; + public const string List = "List"; + public const string Load = "Load"; + public const string Save = "Save"; + public const string Find = "Find"; + + // Other + public const string Error = "Error"; + public const string Warning = "Warning"; + + public const string File = "File"; + public const string Directory = "Directory"; + + public const string Type = "Type"; + public const string Value = "Value"; + public const string Group = "Group"; + + // General sentences"; + public const string XAddedToY = "{0} added to {1}."; + public const string XRemovedFromY = "{0} removed from {1}."; + public const string XDeletedFromY = "{0} deleted from {1}."; + public const string XSuccessfullyUpdated = "{0} successfully updated."; + + // General errors + /// Invalid + public const string XIsInvalid = "{0} is invalid."; + public const string XYFoundButInvalid = "{0} `{1}` found but is invalid."; + public const string XFoundButInvalid = "`{0}` found but is invalid."; + public const string OperationInvalid = "Operation is invalid."; + public const string OperationXInvalid = "Operation {0} is invalid."; + + /// Not Found + public const string XNotFound = "{0} not found."; + public const string XOrYNotFound = "{0} or {1} not found."; + public const string XOrYNotFoundInZ = "{0} or {1} not found in `{2}`."; + public const string FileNotFound = "File `{0}` not found."; + + /// Does not exist + public const string XDoesNotExist = "{0} does not exist."; + public const string XYDoesNotExist = "{0} `{1}` does not exist."; + + /// Duplicate + public const string MoreThanOneXFound = "More than one {0} found."; + public const string XAlreadyContainsY = "{0} already contains {1}."; + public const string XAlreadyContainsYZ = "{0} already contains {1} `{2}`."; + public const string XAlreadyHasY = "{0} already has {1}."; + public const string XAlreadyHasYZ = "{0} already has {1} `{2}`."; + + /// Other + public const string XWasNotExpected = "{0} was not expected."; + public const string XNotProvided = "{0} not provided."; + public const string SpecifyAtLeastOne = "Please specify at least one {0}."; + public const string CouldNotConnectWithTheServer = "Could not connect with the server."; + + // Command Line Parsing + public const string RequiredArgumentIsInvalid = "Required argument {0} is invalid."; + public const string OptionIsInvalid = "Option {0} is invalid."; + public const string ArgumentIsInvalid = "Argument {0} is invalid."; + public const string RequiredArgumentNotPassed = "Required argument {0} was not provided."; + public const string RequiredCommandNotPassed = "Required command was not provided."; + + // dotnet + /// Project + public const string CouldNotFindAnyProjectInDirectory = "Could not find any project in `{0}`."; + public const string CouldNotFindProjectOrDirectory = "Could not find project or directory `{0}`."; + public const string MoreThanOneProjectInDirectory = "Found more than one project in `{0}`. Please specify which one to use."; + public const string FoundInvalidProject = "Found a project `{0}` but it is invalid."; + public const string InvalidProject = "Invalid project `{0}`."; + + /// Solution + public const string CouldNotFindSolutionIn = "Specified solution file {0} does not exist, or there is no solution file in the directory."; + public const string CouldNotFindSolutionOrDirectory = "Could not find solution or directory `{0}`."; + public const string MoreThanOneSolutionInDirectory = "Found more than one solution file in {0}. Please specify which one to use."; + public const string InvalidSolutionFormatString = "Invalid solution `{0}`. {1}"; // {0} is the solution path, {1} is already localized details on the failure + public const string SolutionDoesNotExist = "Specified solution file {0} does not exist, or there is no solution file in the directory."; + + /// add p2p + public const string ReferenceDoesNotExist = "Reference {0} does not exist."; + public const string ReferenceIsInvalid = "Reference `{0}` is invalid."; + public const string SpecifyAtLeastOneReferenceToAdd = "You must specify at least one reference to add."; + public const string ProjectAlreadyHasAReference = "Project {0} already has a reference `{1}`."; + + /// add package + public const string PackageReferenceDoesNotExist = "Package reference `{0}` does not exist."; + public const string PackageReferenceIsInvalid = "Package reference `{0}` is invalid."; + public const string SpecifyAtLeastOnePackageReferenceToAdd = "You must specify at least one package to add."; + public const string PackageReferenceAddedToTheProject = "Package reference `{0}` added to the project."; + public const string ProjectAlreadyHasAPackageReference = "Project {0} already has a reference `{1}`."; + public const string PleaseSpecifyVersion = "Please specify a version of the package."; + + /// add sln + public const string ProjectDoesNotExist = "Project `{0}` does not exist."; + public const string ProjectIsInvalid = "Project `{0}` is invalid."; + public const string SpecifyAtLeastOneProjectToAdd = "You must specify at least one project to add."; + public const string ProjectAddedToTheSolution = "Project `{0}` added to the solution."; + public const string SolutionAlreadyContainsProject = "Solution {0} already contains project {1}."; + + /// del p2p + public const string ReferenceNotFoundInTheProject = "Specified reference {0} does not exist in project {1}."; + public const string ReferenceRemoved = "Reference `{0}` deleted from the project."; + public const string SpecifyAtLeastOneReferenceToRemove = "You must specify at least one reference to remove."; + public const string ReferenceDeleted = "Reference `{0}` deleted."; + + /// del pkg + public const string PackageReferenceNotFoundInTheProject = "Package reference `{0}` could not be found in the project."; + public const string PackageReferenceRemoved = "Reference `{0}` deleted from the project."; + public const string SpecifyAtLeastOnePackageReferenceToRemove = "You must specify at least one package reference to remove."; + public const string PackageReferenceDeleted = "Package reference `{0}` deleted."; + + /// del sln + public const string ProjectNotFoundInTheSolution = "Project `{0}` could not be found in the solution."; + public const string ProjectRemoved = "Project `{0}` removed from solution."; + public const string SpecifyAtLeastOneProjectToRemove = "You must specify at least one project to remove."; + public const string ProjectDeleted = "Project `{0}` deleted from solution."; + + /// list + public const string NoReferencesFound = "There are no {0} references in project {1}. ;; {0} is the type of the item being requested (project, package, p2p) and {1} is the object operated on (a project file or a solution file). "; + public const string NoProjectsFound = "No projects found in the solution."; + + /// arguments + public const string ArgumentsProjectOrSolutionDescription = "The project or solution to operation on. If a file is not specified, the current directory is searched."; + + /// sln + public const string ArgumentsProjectDescription = "The project file to operate on. If a file is not specified, the command will search the current directory for one."; + public const string ArgumentsSolutionDescription = "Solution file to operate on. If not specified, the command will search the current directory for one."; + public const string CmdSlnFile = "SLN_FILE"; + public const string CmdProjectFile = "PROJECT"; + + /// commands + public const string CmdFramework = "FRAMEWORK"; + + /// update pkg + public const string PleaseSpecifyNewVersion = "Please specify new version of the package."; + public const string PleaseSpecifyWhichPackageToUpdate = "Please specify which package to update."; + public const string NothingToUpdate = "Nothing to update."; + public const string EverythingUpToDate = "Everything is already up-to-date."; + public const string PackageVersionUpdatedTo = "Version of package `{0}` updated to `{1}`."; + public const string PackageVersionUpdated = "Version of package `{0}` updated."; + public const string CouldNotUpdateTheVersion = "Could not update the version of the package `{0}`."; + + /// new + public const string TemplateCreatedSuccessfully = "The template {0} created successfully. Please run \"dotnet restore\" to get started!"; + public const string TemplateInstalledSuccesfully = "The template {0} installed successfully. You can use \"dotnet new {0}\" to get started with the new template."; + public const string TemplateCreateError = "Template {0} could not be created. Error returned was: {1}."; + public const string TemplateInstallError = "Template {0} could not be installed. Error returned was: {1}."; + public const string SpecifiedNameExists = "Specified name {0} already exists. Please specify a different name."; + public const string SpecifiedAliasExists = "Specified alias {0} already exists. Please specify a different alias."; + public const string MandatoryParameterMissing = "Mandatory parameter {0} missing for template {1}. "; + + public const string ProjectNotCompatibleWithFrameworks = "Project `{0}` cannot be added due to incompatible targeted frameworks between the two projects. Please review the project you are trying to add and verify that is compatible with the following targets:"; + public const string ProjectDoesNotTargetFramework = "Project `{0}` does not target framework `{1}`."; + public const string ProjectCouldNotBeEvaluated = "Project `{0}` could not be evaluated. Evaluation failed with following error:\n{1}"; + } +} diff --git a/src/dotnet-archive/Program.cs b/src/dotnet-archive/Program.cs new file mode 100644 index 0000000000..3376bf47ef --- /dev/null +++ b/src/dotnet-archive/Program.cs @@ -0,0 +1,96 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.DotNet.Cli.CommandLine; +//using Microsoft.DotNet.Cli.Utils; +using Microsoft.DotNet.Archive; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Reflection; + +namespace Microsoft.DotNet.Tools.Archive +{ + + public partial class ArchiveCommand + { + public static int Main(string[] args) + { + //DebugHelper.HandleDebugSwitch(ref args); + + var app = new CommandLineApplication(); + app.Name = "archive"; + app.FullName = ".NET archiver"; + app.Description = "Archives and expands sets of files"; + app.HelpOption("-h|--help"); + + var extract = app.Option("-x|--extract ", "Directory to extract to", CommandOptionType.SingleValue); + var archiveFile = app.Option("-a|--archive ", "Archive to operate on", CommandOptionType.SingleValue); + var externals = app.Option("--external ...", "External files and directories to consider for extraction", CommandOptionType.MultipleValue); + var sources = app.Argument("...", "Files & directory to include in the archive", multipleValues:true); + + app.OnExecute(() => { + + if (extract.HasValue() && sources.Values.Any()) + { + Console.WriteLine("Extract '-x' can only be specified when no '' are specified to add to the archive."); + return 1; + } + else if (!extract.HasValue() && !sources.Values.Any()) + { + Console.WriteLine("Either extract '-x' or '' must be specified."); + return 1; + } + + if (!archiveFile.HasValue()) + { + Console.WriteLine("Archive '-a' must be specified."); + return 1; + } + + var progress = new ConsoleProgressReport(); + + var archive = new IndexedArchive(); + foreach (var external in externals.Values) + { + if (Directory.Exists(external)) + { + archive.AddExternalDirectory(external); + } + else + { + archive.AddExternalFile(external); + } + } + + if (sources.Values.Any()) + { + foreach(var source in sources.Values) + { + if (Directory.Exists(source)) + { + archive.AddDirectory(source, progress); + } + else + { + archive.AddFile(source, Path.GetFileName(source)); + } + } + + archive.Save(archiveFile.Value(), progress); + } + else // sources not specified, extract must have been specified + { + archive.Extract(archiveFile.Value(), extract.Value(), progress); + + } + + return 0; + }); + + return app.Execute(args); + } + } +} diff --git a/src/dotnet-archive/dotnet-archive.csproj b/src/dotnet-archive/dotnet-archive.csproj new file mode 100644 index 0000000000..67a2ef1a9b --- /dev/null +++ b/src/dotnet-archive/dotnet-archive.csproj @@ -0,0 +1,15 @@ + + + + + + netcoreapp2.0 + Exe + false + + + + + + +